"""Fetch and extract text content from URLs for AI processing.

Adapted from Муравей. Uses stdlib urllib (no requests dependency).
Security: URL validation via src.utils.security.validate_url.
"""

from __future__ import annotations

import logging
import re
from html import unescape as _html_unescape

log = logging.getLogger(__name__)

_USER_AGENT = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/120.0.0.0 Safari/537.36"
)


def fetch_url_text(url: str, max_chars: int = 30000) -> str:
    """Fetch a URL and extract readable text content.

    Returns plain text extracted from HTML, or raw text for non-HTML responses.
    Raises ValueError on failure.
    """
    from .utils.security import validate_url

    # Block non-HTTP schemes before normalization
    if url.startswith(("file://", "ftp://", "data:", "javascript:")):
        raise ValueError(f"Invalid or unsafe URL: {url[:100]}")

    # Normalise bare domains (e.g. "example.com" → "https://example.com")
    if not url.startswith(("http://", "https://")):
        url = "https://" + url

    if not validate_url(url, allowed_schemes=["http", "https"]):
        raise ValueError(f"Invalid or unsafe URL: {url[:100]}")

    try:
        import urllib.request
        from .utils.proxied_urllib import proxied_urlopen

        req = urllib.request.Request(
            url,
            headers={
                "User-Agent": _USER_AGENT,
                "Accept": "text/html,application/xhtml+xml,text/plain,*/*",
            },
        )
        with proxied_urlopen(req, timeout=15) as resp:
            content_type = resp.headers.get("Content-Type", "")
            raw = resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        # Sanitize error: strip URL query params that might contain API keys
        err_msg = str(e)
        if "?" in err_msg:
            err_msg = err_msg[: err_msg.index("?")] + "?..."
        raise ValueError(f"Failed to fetch URL: {err_msg}") from e

    if "text/html" in content_type or "<html" in raw[:500].lower():
        text = _html_to_text(raw)
    else:
        text = raw

    if len(text) > max_chars:
        text = text[:max_chars] + "\n\n[... truncated]"

    log.info("Fetched %d chars from %s", len(text), url[:60])
    return text.strip()


def _html_to_text(html: str) -> str:
    """Convert HTML to readable plain text using regex (no external dependency)."""
    # Remove script, style, nav, footer, header blocks
    html = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r"<style[^>]*>.*?</style>", "", html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r"<nav[^>]*>.*?</nav>", "", html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r"<footer[^>]*>.*?</footer>", "", html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r"<header[^>]*>.*?</header>", "", html, flags=re.DOTALL | re.IGNORECASE)

    # Convert common elements to text
    html = re.sub(r"<br\s*/?>", "\n", html, flags=re.IGNORECASE)
    html = re.sub(r"</?p[^>]*>", "\n", html, flags=re.IGNORECASE)
    html = re.sub(r"</?div[^>]*>", "\n", html, flags=re.IGNORECASE)
    html = re.sub(
        r"<h[1-6][^>]*>(.*?)</h[1-6]>", r"\n## \1\n", html, flags=re.DOTALL | re.IGNORECASE
    )
    html = re.sub(r"<li[^>]*>(.*?)</li>", r"\n- \1", html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(
        r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>',
        r"\2 (\1)",
        html,
        flags=re.DOTALL | re.IGNORECASE,
    )

    # Remove all remaining HTML tags
    text = re.sub(r"<[^>]+>", "", html)

    # Decode HTML entities
    text = _html_unescape(text)
    text = text.replace("\xa0", " ")

    # Clean up whitespace
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = re.sub(r" {2,}", " ", text)

    return text.strip()


def is_url(text: str) -> bool:
    """Check if a string looks like a URL."""
    text = text.strip()
    if text.startswith(("http://", "https://", "www.")):
        return True
    if re.match(r"^[\w.-]+\.\w{2,}(/\S*)?$", text):
        return True
    return False
