link-analyzer/bot.py

#!/usr/bin/env python3
"""
Discord bot for #remora channel - analyzes links in real-time with web_fetch + AI
Posts summaries, adds to Tududi inbox, maintains JSON history + logs
"""

import discord
import os
import json
import re
import requests
from datetime import datetime
from pathlib import Path
from dotenv import load_dotenv
import logging
from urllib.parse import urlparse

# Load .env file
load_dotenv()

# Setup logging
log_file = Path(__file__).parent / "bot.log"
logging.basicConfig(
    level=logging.DEBUG,
    format='[%(asctime)s] [%(levelname)-8s] %(message)s',
    handlers=[
        logging.FileHandler(log_file),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

# Config
CHANNEL_ID = 1467557082583535729
TRACKER_FILE = Path(__file__).parent / "tracker.json"
TUDUDI_API_URL = os.getenv("TUDUDI_API_URL", "https://todo.dilain.com/api/v1")
TUDUDI_API_KEY = os.getenv("TUDUDI_API_KEY")
GATEWAY_URL = os.getenv("OPENCLAW_GATEWAY", "http://127.0.0.1:18789")
GATEWAY_TOKEN = os.getenv("OPENCLAW_GATEWAY_TOKEN", "")

logger.info("=" * 60)
logger.info("Bot startup")
logger.info(f"  Channel ID: {CHANNEL_ID}")
logger.info(f"  Tududi API: {TUDUDI_API_URL}")
logger.info(f"  Gateway: {GATEWAY_URL}")
logger.info("=" * 60)

# Load or init tracker
def load_tracker():
    if TRACKER_FILE.exists():
        with open(TRACKER_FILE) as f:
            return json.load(f)
    return {
        "channel_id": CHANNEL_ID,
        "processed_message_ids": [],
        "links": []
    }

def save_tracker(data):
    with open(TRACKER_FILE, "w") as f:
        json.dump(data, f, indent=2)

# Detect links in text
def extract_urls(text):
    url_pattern = r'https?://[^\s<>"{}|\\^`\[\]]+'
    return re.findall(url_pattern, text)

# Detect link type
def detect_link_type(url):
    domain = urlparse(url).netloc.lower()
    
    if "github.com" in domain:
        return "GitHub"
    elif "reddit.com" in domain:
        return "Reddit"
    elif "youtube.com" in domain or "youtu.be" in domain:
        return "YouTube"
    elif "tiktok.com" in domain:
        return "TikTok"
    elif "twitter.com" in domain or "x.com" in domain:
        return "Twitter/X"
    elif "medium.com" in domain:
        return "Medium"
    elif "dev.to" in domain:
        return "Dev.to"
    elif "arxiv.org" in domain:
        return "arXiv"
    else:
        return "Article"

# Fetch URL content using requests
def fetch_url_content(url):
    """Fetch URL and return title + excerpt"""
    logger.debug(f"  📥 Fetching: {url}")
    
    try:
        response = requests.get(
            url,
            timeout=5,
            headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)'},
            allow_redirects=True
        )
        response.raise_for_status()
        content = response.text[:3000]  # First 3k chars
        
        # Extract title
        title_match = re.search(r'<title[^>]*>([^<]+)</title>', content, re.IGNORECASE)
        title = title_match.group(1).strip() if title_match else "No title found"
        
        # Extract meta description
        desc_match = re.search(r'<meta\s+name="description"\s+content="([^"]+)"', content, re.IGNORECASE)
        description = desc_match.group(1) if desc_match else ""
        
        logger.debug(f"    ✓ Fetched: {title}")
        return {
            "title": title,
            "description": description,
            "content": content,
            "status": "ok"
        }
    except requests.Timeout:
        logger.warning(f"    ⏱️ Timeout: {url}")
        return {"title": "Request timeout", "status": "timeout", "content": ""}
    except requests.HTTPError as e:
        logger.warning(f"    ❌ HTTP {e.response.status_code}: {url}")
        return {"title": f"HTTP {e.response.status_code}", "status": "http_error", "content": ""}
    except Exception as e:
        logger.error(f"    ❌ Error: {e}")
        return {"title": "Fetch failed", "status": "error", "error": str(e), "content": ""}

# Analyze with OpenClaw gateway (Haiku)
def analyze_with_gateway(url, title, content):
    """Send to OpenClaw gateway for AI analysis"""
    logger.debug(f"  🤖 Analyzing with gateway: {url}")
    
    # Build analysis prompt
    prompt = f"""Analyze this webpage briefly (2-3 sentences max):

URL: {url}
Title: {title}

Content excerpt:
{content[:1000]}

Provide:
1. What is this about? (1 sentence)
2. Who should read this? (optional)
3. Suggested Tududi tag (e.g., "to-read", "learning", "inspiration", "tool")

Keep it concise!"""
    
    try:
        response = requests.post(
            f"{GATEWAY_URL}/v1/messages",
            headers={"Authorization": f"Bearer {GATEWAY_TOKEN}"} if GATEWAY_TOKEN else {},
            json={
                "messages": [{"role": "user", "content": prompt}],
                "model": "openrouter/anthropic/claude-haiku-4.5",
                "max_tokens": 150
            },
            timeout=10
        )
        
        if response.status_code == 200:
            result = response.json()
            analysis = result.get("content", [{}])[0].get("text", "")
            logger.debug(f"    ✓ Analysis complete")
            return analysis
        else:
            logger.warning(f"    Gateway error: {response.status_code}")
            return None
    except Exception as e:
        logger.warning(f"    Gateway timeout/error: {e}")
        return None

# Send to Tududi inbox
def add_to_tududi(title, url, link_type, analysis=""):
    """Add to Tududi inbox with summary"""
    logger.debug(f"  📌 Adding to Tududi: {title}")
    
    try:
        if not TUDUDI_API_KEY:
            logger.warning("    TUDUDI_API_KEY not set")
            return False
        
        content = f"📌 **{link_type}**: {title}\n🔗 {url}"
        if analysis:
            content += f"\n\n💡 Summary:\n{analysis}"
        
        response = requests.post(
            f"{TUDUDI_API_URL}/inbox",
            headers={
                "Authorization": f"Bearer {TUDUDI_API_KEY}",
                "Content-Type": "application/json"
            },
            json={"content": content},
            timeout=5
        )
        
        if response.status_code == 200:
            logger.info(f"    ✓ Added to Tududi inbox")
            return True
        else:
            logger.warning(f"    Tududi error: {response.status_code}")
            return False
    except Exception as e:
        logger.error(f"    Tududi error: {e}")
        return False

# Discord bot
intents = discord.Intents.default()
intents.message_content = True

class LinkAnalyzerBot(discord.Client):
    async def on_ready(self):
        logger.info(f"✅ Bot logged in as {self.user}")
        logger.info(f"📍 Watching channel #remora ({CHANNEL_ID})")

    async def on_message(self, message):
        # Ignore bot's own messages
        if message.author == self.user:
            return
        
        # Only process #remora channel
        if message.channel.id != CHANNEL_ID:
            return
        
        # Check for URLs
        urls = extract_urls(message.content)
        if not urls:
            logger.debug(f"No URLs in message from {message.author}")
            return
        
        # Skip if already processed
        tracker = load_tracker()
        if message.id in tracker["processed_message_ids"]:
            logger.debug(f"Skipping already-processed message {message.id}")
            return
        
        logger.info(f"🔗 New link(s) from {message.author}: {message.content}")
        
        # Process each URL
        for url in urls:
            try:
                logger.info(f"Processing: {url}")
                link_type = detect_link_type(url)
                
                # Fetch content
                fetch_result = fetch_url_content(url)
                title = fetch_result["title"]
                
                # Analyze with gateway
                analysis = None
                if fetch_result["status"] == "ok":
                    analysis = analyze_with_gateway(url, title, fetch_result.get("content", ""))
                
                # Add to Tududi
                tududi_ok = add_to_tududi(title, url, link_type, analysis or "")
                
                # Format response for Discord
                response_text = f"📌 **{link_type}**: {title}"
                if analysis:
                    # Truncate to 200 chars for Discord
                    summary = analysis[:200].split('\n')[0]
                    response_text += f"\n💡 {summary}"
                
                logger.debug(f"Posting response: {response_text}")
                
                # Post in channel
                await message.reply(response_text, mention_author=False)
                
                # Update tracker
                tracker["links"].append({
                    "url": url,
                    "title": title,
                    "type": link_type,
                    "author": str(message.author),
                    "message_id": message.id,
                    "date": datetime.now().isoformat(),
                    "analysis": analysis,
                    "tududi": tududi_ok,
                    "fetch_status": fetch_result["status"]
                })
                
                logger.info(f"✓ Processed: {url}")
                
            except Exception as e:
                logger.error(f"❌ Error processing {url}: {e}")
                await message.reply(f"❌ Error analyzing link: {e}", mention_author=False)
        
        # Update processed IDs
        tracker["processed_message_ids"].append(message.id)
        save_tracker(tracker)
        logger.info(f"Updated tracker, total links: {len(tracker['links'])}")

# Main
if __name__ == "__main__":
    token = os.getenv("DISCORD_BOT_TOKEN")
    if not token:
        logger.error("❌ DISCORD_BOT_TOKEN not set!")
        exit(1)
    
    logger.info("Starting bot...")
    bot = LinkAnalyzerBot(intents=intents)
    bot.run(token)