#!/usr/bin/env python3
"""
web-scrape.py — Extract clean readable text from web pages.

Saves tokens by converting raw HTML (~10-50K chars) into clean text
(~1-5K chars) before the LLM sees it. That's 4-10x fewer tokens.

Usage:
  python3 web-scrape.py --url https://example.com/article
  python3 web-scrape.py --url https://example.com --format markdown --max-chars 5000
  cat urls.txt | python3 web-scrape.py --urls-from-stdin

Returns JSON to stdout. One result per URL.

Environment:
  USER_AGENT    - Custom User-Agent string (optional)

Token savings: 4-10x reduction in tokens vs raw HTML.
"""

import argparse
import json
import os
import re
import sys
import urllib.parse
import urllib.request
from html.parser import HTMLParser
from io import StringIO


try:
    from bs4 import BeautifulSoup
    HAS_BS4 = True
except ImportError:
    HAS_BS4 = False


# ─── HTML to Text ──────────────────────────────────

class MLStripper(HTMLParser):
    """Strip HTML tags, preserve text."""
    def __init__(self):
        super().__init__()
        self.reset()
        self.strict = False
        self.convert_charrefs = True
        self.text = StringIO()

    def handle_data(self, data):
        self.text.write(data)

    def get_text(self):
        return self.text.getvalue()


def strip_html(html):
    """Remove HTML tags using stdlib only."""
    s = MLStripper()
    s.feed(html)
    return s.get_text()


def clean_text(text):
    """Normalize whitespace and clean up extracted text."""
    # Remove excessive whitespace
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'^\s+', '', text, flags=re.MULTILINE)
    text = text.strip()
    return text


def extract_text_bs4(html, selector=None):
    """Extract using BeautifulSoup (best results)."""
    soup = BeautifulSoup(html, 'html.parser')

    # Remove noise elements
    for tag in soup(['script', 'style', 'nav', 'footer', 'header',
                     'noscript', 'iframe', 'svg', 'form', 'button']):
        tag.decompose()

    if selector:
        selected = soup.select(selector)
        if not selected:
            return None, "Selector matched no elements"
        text = '\n\n'.join(s.get_text(strip=True) for s in selected)
    else:
        # Try article/main content first
        main = (soup.find('article') or soup.find('main')
                or soup.find(class_=re.compile(r'content|post|article|entry'))
                or soup.body or soup)
        text = main.get_text(separator='\n', strip=True)

    text = clean_text(text)
    title = soup.title.string.strip() if soup.title and soup.title.string else ""
    return text, title


def extract_text_fallback(html):
    """Extract using stdlib HTMLParser (no bs4)."""
    # Try to find title
    title_match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
    title = title_match.group(1).strip() if title_match else ""

    text = strip_html(html)
    text = clean_text(text)
    return text, title


MAX_DOWNLOAD_BYTES = 5 * 1024 * 1024  # 5 MiB hard cap for local agent safety


def fetch_url(url, timeout=15):
    """Fetch a URL and return HTML content."""
    parsed = urllib.parse.urlparse(url)
    if parsed.scheme not in ('http', 'https'):
        raise ValueError(f"Only http/https URLs are supported (got {parsed.scheme or 'none'})")
    if not parsed.netloc:
        raise ValueError("URL is missing a host")

    headers = {
        'User-Agent': os.environ.get('USER_AGENT',
            'Mozilla/5.0 (compatible; MediaToolBot/1.0; +https://agentmediatools.com)'),
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
    }
    req = urllib.request.Request(url, headers=headers, method='GET')
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        # Bound download size so hostile pages cannot OOM the agent host.
        chunks = []
        total = 0
        while True:
            chunk = resp.read(64 * 1024)
            if not chunk:
                break
            total += len(chunk)
            if total > MAX_DOWNLOAD_BYTES:
                raise ValueError(f"Response exceeds {MAX_DOWNLOAD_BYTES} byte download limit")
            chunks.append(chunk)
        html_bytes = b''.join(chunks)
        # Try to detect encoding
        charset = resp.headers.get_content_charset()
        if charset:
            html = html_bytes.decode(charset, errors='replace')
        else:
            html = html_bytes.decode('utf-8', errors='replace')
    return html, len(html_bytes)


# ─── CLI ────────────────────────────────────────────

def parse_args():
    parser = argparse.ArgumentParser(
        description="Extract clean readable text from web pages. Saves 4-10x tokens vs raw HTML.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python3 web-scrape.py --url https://example.com/article
  python3 web-scrape.py --url https://example.com --format markdown --max-chars 5000
  cat urls.txt | python3 web-scrape.py --urls-from-stdin --format json

Returns JSON to stdout.
"""
    )
    parser.add_argument("--url", type=str, help="URL to scrape")
    parser.add_argument("--selector", type=str,
        help="CSS selector for targeted extraction (requires BeautifulSoup)")
    parser.add_argument("--format", type=str, choices=["text", "markdown", "json"], default="json",
        help="Output format (json recommended for agents)")
    parser.add_argument("--max-chars", type=int, default=10000,
        help="Maximum characters to return (default: 10000)")
    parser.add_argument("--urls-from-stdin", action="store_true",
        help="Read URLs from stdin, one per line")

    args = parser.parse_args()

    if not args.url and not args.urls_from_stdin:
        parser.error("Either --url or --urls-from-stdin is required")
    if args.urls_from_stdin and sys.stdin.isatty():
        parser.error("--urls-from-stdin requires piped input, not a TTY")

    return args


def main():
    args = parse_args()

    urls = []
    if args.urls_from_stdin:
        urls = [line.strip() for line in sys.stdin if line.strip()]
    elif args.url:
        urls = [args.url]

    results = []

    for url in urls:
        try:
            raw_html, raw_size = fetch_url(url)

            if HAS_BS4:
                text, title = extract_text_bs4(raw_html, args.selector)
                if text is None:
                    text, title = extract_text_fallback(raw_html)
            else:
                text, title = extract_text_fallback(raw_html)

            # Truncate to max-chars
            truncated = len(text) > args.max_chars
            text = text[:args.max_chars]

            chars_removed = raw_size - len(text)
            tokens_saved = int(chars_removed / 4)  # ~4 chars per token

            result = {
                "success": True,
                "url": url,
                "title": title,
                "word_count": len(text.split()),
                "char_count": len(text),
                "raw_char_count": raw_size,
                "chars_removed": chars_removed,
                "tokens_saved": tokens_saved,
                "truncated": truncated,
                "text": text,
                "method": "beautifulsoup4" if HAS_BS4 else "stdlib_fallback",
            }

            print(f"[{url[:60]}] {raw_size:,} raw → {len(text):,} chars (saved ~{tokens_saved:,} tokens)",
                  file=sys.stderr)

        except (urllib.error.URLError, urllib.error.HTTPError, OSError, ValueError) as e:
            result = {
                "success": False,
                "url": url,
                "error": str(e),
            }
            print(f"[{url[:60]}] ❌ Error: {e}", file=sys.stderr)

        results.append(result)

    if args.format == "json":
        output = {
            "success": len([r for r in results if r.get("success")]) > 0,
            "total_tokens_saved": sum(r.get("tokens_saved", 0) for r in results),
            "results": results,
        }

        # Single-URL convenience fields (keep url/title/text for agents)
        if len(results) == 1 and results[0].get("success"):
            r0 = results[0]
            output["url"] = r0.get("url", "")
            output["title"] = r0.get("title", "")
            output["text"] = r0.get("text", "")
            del output["results"]

        print(json.dumps(output, indent=2))
    elif args.format == "text":
        for r in results:
            if r.get("success"):
                print(r["text"])
            else:
                print(f"Error: {r.get('error', 'unknown')}")
    elif args.format == "markdown":
        for r in results:
            if r.get("success"):
                print(f"# {r['title']}\n\n{r['text']}")
            else:
                print(f"# Error\n\n{r.get('error', 'unknown')}")


if __name__ == "__main__":
    main()
