"""
Content reader: fetch candidate pages in parallel and extract clean,
main-body text. This is what makes the answers *deep* instead of a
rehash of SERP snippets.

Extraction ladder (first success wins):
  1. trafilatura  - best-in-class boilerplate removal
  2. selectolax   - fast structural fallback (article/main/content nodes)
  3. raw text     - last resort
Special-cased fast paths exist for Wikipedia and arXiv APIs.
"""
from __future__ import annotations

import asyncio
import logging
import re
from urllib.parse import urlparse
from typing import Any, List, Optional, Tuple

from selectolax.parser import HTMLParser

from .config import settings
from .core import (TTLCache, base_headers, fetch_json, fetch_text,
                    gather_capped, page_cache)
from .schemas import Chunk, Document, canonical_url, domain_of
from .structured import extract_structured, structure_score
from .verbatim import extract_verbatim

log = logging.getLogger("deepsearch.reader")

try:
    import trafilatura  # type: ignore
    _HAS_TRAFILATURA = True
except Exception:  # pragma: no cover
    _HAS_TRAFILATURA = False

_JUNK_TAGS = (
    "script", "style", "nav", "header", "footer", "aside", "form", "noscript",
    "iframe", "svg", "button", "figure figcaption", ".advertisement", ".ads",
    ".cookie", ".newsletter", ".sidebar", ".related", ".comments", ".social",
    ".breadcrumb", ".menu", ".popup", ".modal", "[role=navigation]",
    "[role=banner]", "[role=complementary]",
)

_CONTENT_SELECTORS = (
    "article", "main", '[role="main"]', ".post-content", ".entry-content",
    ".article-body", ".article-content", ".story-body", ".content-body",
    "#content", ".content", ".markdown-body", ".documentation", ".post",
)

_BOILERPLATE = re.compile(
    r"(cookie policy|accept all cookies|subscribe to our newsletter|"
    r"sign up for free|all rights reserved|terms of service|privacy policy|"
    r"share this article|advertisement|related articles)",
    re.I,
)


def _collapse(text: str) -> str:
    text = re.sub(r"[ \t\xa0]+", " ", text or "")
    text = re.sub(r"\n{3,}", "\n\n", text)
    lines = []
    for ln in text.split("\n"):
        s = ln.strip()
        if not s:
            continue
        # drop nav-ish one-word lines and boilerplate
        if len(s) < 3:
            continue
        if _BOILERPLATE.search(s) and len(s) < 160:
            continue
        lines.append(s)
    return "\n".join(lines).strip()


def _extract_selectolax(html: str) -> str:
    try:
        tree = HTMLParser(html)
    except Exception:
        return ""
    for sel in _JUNK_TAGS:
        try:
            for node in tree.css(sel):
                node.decompose()
        except Exception:
            continue
    best = ""
    for sel in _CONTENT_SELECTORS:
        try:
            nodes = tree.css(sel)
        except Exception:
            continue
        for n in nodes:
            t = n.text(separator="\n", strip=True) or ""
            if len(t) > len(best):
                best = t
    if len(best) < 400 and tree.body:
        body = tree.body.text(separator="\n", strip=True) or ""
        if len(body) > len(best):
            best = body
    return best


def _extract_title(html: str) -> str:
    try:
        tree = HTMLParser(html)
        for sel in ('meta[property="og:title"]', 'meta[name="twitter:title"]'):
            n = tree.css_first(sel)
            if n and n.attributes.get("content"):
                return " ".join(n.attributes["content"].split())[:300]
        if tree.css_first("h1"):
            return " ".join(tree.css_first("h1").text().split())[:300]
        if tree.css_first("title"):
            return " ".join(tree.css_first("title").text().split())[:300]
    except Exception:
        pass
    return ""


def _extract_date(html: str) -> Optional[str]:
    try:
        tree = HTMLParser(html)
        for sel, attr in (
            ('meta[property="article:published_time"]', "content"),
            ('meta[name="publish-date"]', "content"),
            ('meta[name="date"]', "content"),
            ('meta[itemprop="datePublished"]', "content"),
            ("time[datetime]", "datetime"),
        ):
            n = tree.css_first(sel)
            if n and n.attributes.get(attr):
                return n.attributes[attr][:32]
    except Exception:
        pass
    m = re.search(r"(20[0-2]\d)[-/](0?[1-9]|1[0-2])[-/](0?[1-9]|[12]\d|3[01])", html or "")
    return m.group(0) if m else None


def extract_content(html: str, url: str = "") -> Tuple[str, str, Optional[str]]:
    """Return (content, title, published)."""
    if not html:
        return "", "", None
    text = ""
    if _HAS_TRAFILATURA:
        try:
            text = trafilatura.extract(
                html, include_comments=False, include_tables=True,
                no_fallback=False, favor_precision=False, url=url or None,
            ) or ""
        except Exception:
            text = ""
    if len(text) < 300:
        alt = _extract_selectolax(html)
        if len(alt) > len(text):
            text = alt
    return _collapse(text), _extract_title(html), _extract_date(html)


# --------------------------------------------------------------------------
# API fast paths - cheaper and cleaner than scraping the rendered page
# --------------------------------------------------------------------------
async def _read_wikipedia(url: str) -> Optional[str]:
    m = re.search(r"/wiki/([^#?]+)", url)
    if not m:
        return None
    data = await fetch_json(
        "https://en.wikipedia.org/w/api.php",
        params={"action": "query", "prop": "extracts", "explaintext": 1,
                "redirects": 1, "format": "json", "titles": m.group(1).replace("_", " ")},
        timeout=settings.scrape_timeout,
    )
    if not isinstance(data, dict):
        return None
    pages = data.get("query", {}).get("pages", {})
    for _, page in pages.items():
        ex = page.get("extract")
        if ex:
            return _collapse(ex)
    return None


async def _read_arxiv(url: str) -> Optional[str]:
    m = re.search(r"arxiv\.org/(?:abs|pdf)/([\w.\-/]+?)(?:v\d+)?(?:\.pdf)?$", url)
    if not m:
        return None
    xml = await fetch_text(
        "https://export.arxiv.org/api/query",
        params={"id_list": m.group(1), "max_results": 1},
        timeout=settings.scrape_timeout,
    )
    if not xml:
        return None
    s = re.search(r"<summary[^>]*>(.*?)</summary>", xml, re.S)
    t = re.search(r"<title[^>]*>(.*?)</title>", xml, re.S)
    if not s:
        return None
    body = re.sub(r"<[^>]+>", "", s.group(1))
    head = re.sub(r"<[^>]+>", "", t.group(1)) if t else ""
    return _collapse(f"{head}\n\n{body}")


# --------------------------------------------------------------------------
# public reader API
# --------------------------------------------------------------------------
async def read_document(doc: Document) -> Document:
    """Fetch + extract a single document in place."""
    key = TTLCache.key("page", doc.url)
    cached = page_cache.get(key)
    if cached is not None:
        content, title, pub = cached
    else:
        content, title, pub = "", "", None
        try:
            if "wikipedia.org/wiki/" in doc.url:
                content = await _read_wikipedia(doc.url) or ""
            elif "arxiv.org/" in doc.url:
                content = await _read_arxiv(doc.url) or ""
            if len(content) < 300:
                html = await fetch_text(
                    doc.url, timeout=settings.scrape_timeout,
                    max_bytes=settings.max_page_bytes,
                )
                if html:
                    c2, title, pub = extract_content(html, doc.url)
                    if len(c2) > len(content):
                        content = c2
        except Exception as e:  # noqa: BLE001
            doc.fetch_error = str(e)[:120]
        page_cache.set(key, (content, title, pub), settings.page_cache_ttl)

    if content:
        doc.content = content[:60000]
        doc.fetched = True
        doc.word_count = len(content.split())
        if title and (not doc.title or len(doc.title) < 12):
            doc.title = title
        if pub and not doc.published:
            doc.published = pub
    elif not doc.fetch_error:
        doc.fetch_error = "no content extracted"
    return doc


async def read_documents(
    docs: List[Document],
    limit: Optional[int] = None,
    deadline: Optional[float] = None,
    on_event: Optional[Any] = None,
) -> List[Document]:
    """
    Read many documents concurrently, honouring a wall-clock deadline.

    `on_event(kind, doc)` is invoked as each page starts and finishes so the
    caller can surface genuine per-site progress ("reading nature.com",
    "read 1,240 words") instead of a fake placeholder ticker.
    """
    if not docs:
        return []
    targets = docs[: limit or len(docs)]

    async def _one(d: Document) -> Document:
        if on_event:
            try:
                await on_event("start", d)
            except Exception:
                pass
        await read_document(d)
        if on_event:
            try:
                await on_event("done", d)
            except Exception:
                pass
        return d

    await gather_capped(
        (_one(d) for d in targets),
        limit=settings.scrape_concurrency,
        deadline=deadline,
    )
    return targets


# --------------------------------------------------------------------------
# chunking - split read pages into passages for evidence selection
# --------------------------------------------------------------------------
def chunk_document(
    doc: Document, target_chars: int = 1100, overlap: int = 120, max_chunks: int = 8
) -> List[Chunk]:
    text = doc.content or doc.snippet
    if not text:
        return []
    paras = [p.strip() for p in text.split("\n") if len(p.strip()) > 40]
    if not paras:
        paras = [text]
    chunks: List[Chunk] = []
    buf = ""
    for p in paras:
        if len(buf) + len(p) + 1 <= target_chars:
            buf = f"{buf}\n{p}" if buf else p
        else:
            if buf:
                chunks.append(Chunk(text=buf, doc=doc, position=len(chunks)))
                buf = (buf[-overlap:] + "\n" + p) if overlap else p
            else:
                chunks.append(Chunk(text=p[:target_chars], doc=doc, position=len(chunks)))
                buf = ""
        if len(chunks) >= max_chunks:
            break
    if buf and len(chunks) < max_chunks:
        chunks.append(Chunk(text=buf, doc=doc, position=len(chunks)))
    return chunks


# --------------------------------------------------------------------------
# deep single-page / single-site reading
# --------------------------------------------------------------------------
async def read_full_page(url: str, max_chars: int = 120_000) -> Tuple[str, str]:
    """
    Read one page as completely as possible.

    Ordinary reading caps extraction because a hundred pages are in flight.
    Here exactly one page matters, so we take the whole body and fall back to
    a raw-text sweep when the boilerplate remover is too aggressive - song
    lyrics, recipes and tables are frequent victims of that.
    """
    html = await fetch_text(url, timeout=settings.scrape_timeout + 6,
                            max_bytes=settings.max_page_bytes)

    # Content hosts (lyrics, recipes, transcripts) routinely block datacenter
    # IPs outright. Focus mode is a single deliberate page, so it is worth
    # retrying through the rotating relay - direct fetches stay unproxied.
    if not html or len(html) < 2000:
        try:
            from tecox.rewind import prxbin
            from .core import get_client
            client = await get_client()
            for country in ("us", "gb", "de"):
                data, status = await prxbin(client, url, "GET",
                                            base_headers(), None, country,
                                            timeout=70.0)
                body = data if isinstance(data, str) else ""
                if status == 200 and len(body) > len(html or ""):
                    html = body
                    break
        except Exception:
            pass

    if not html:
        return "", ""

    # Structured verbatim content (lyrics, recipes, transcripts) first: the
    # prose extractor treats short repeated lines as boilerplate and deletes
    # exactly what the user asked to see.
    verb = extract_verbatim(html)
    text, title, _pub = extract_content(html, url)

    # Structure-preserving pass: keeps code fences, headings, steps and
    # tables that the prose extractor flattens into a paragraph. Used when
    # it recovers meaningfully more structure or content.
    struct = extract_structured(html, url)
    if struct and (structure_score(struct) >= 4 or len(struct) > len(text) * 1.2):
        text = struct

    # verbatim.py is a fallback for sites whose content lives in a custom
    # container the generic extractor misses. It only wins when it
    # actually recovers more than everything else did.
    if verb:
        vtext, _kind = verb
        if vtext and vtext not in text:
            text = f"{vtext}\n\n{text}" if text else vtext

    if len(text) < 1500:
        try:
            tree = HTMLParser(html)
            for sel in ("script", "style", "nav", "header", "footer", "aside",
                        "noscript", "iframe", "svg", "form"):
                for n in tree.css(sel):
                    n.decompose()
            body = tree.body.text(separator="\n", strip=True) if tree.body else ""
            if len(body) > len(text):
                text = _collapse(body)
        except Exception:
            pass
    return text[:max_chars], title


async def read_site_deep(
    site: str, query: str, max_pages: int = 6,
    seed_urls: Optional[List[str]] = None,
) -> List[Document]:
    """
    Read a single site thoroughly.

    Starts from any seed URLs (e.g. the source we cited last turn), then
    follows on-site links whose anchor text or path overlaps the query. Only
    `site` is ever fetched - this never wanders off-domain.
    """
    site = (site or "").lower().lstrip("www.")
    if not site:
        return []

    terms = {w for w in re.findall(r"[a-z0-9]{3,}", (query or "").lower())}
    seen: set = set()
    queue: List[str] = []

    for u in (seed_urls or []):
        cu = canonical_url(u)
        if site in cu and cu not in seen:
            seen.add(cu)
            queue.append(cu)

    # No seeds: search engines frequently ignore `site:`, so start from the
    # site itself and let link discovery find the relevant pages.
    if not queue:
        for entry in (f"https://{site}/", f"https://{site}/docs/",
                      f"https://{site}/tutorial/", f"https://{site}/blog/"):
            cu = canonical_url(entry)
            if cu not in seen:
                seen.add(cu)
                queue.append(cu)

    docs: List[Document] = []

    async def take(url: str) -> Optional[Document]:
        text, title = await read_full_page(url)
        if len(text) < 200:
            return None
        d = Document(url=url, title=title or url, domain=domain_of(url))
        d.content = text
        d.fetched = True
        d.word_count = len(text.split())
        return d

    # first wave: the seeds
    if queue:
        got = await gather_capped((take(u) for u in queue[:max_pages]),
                                  limit=settings.scrape_concurrency)
        docs.extend([d for d in got if d])

    # second wave: relevant on-site links discovered from what we have
    if len(docs) < max_pages:
        links: List[Tuple[int, str]] = []
        for d in docs:
            html = await fetch_text(d.url, timeout=settings.scrape_timeout)
            if not html:
                continue
            try:
                tree = HTMLParser(html)
            except Exception:
                continue
            for a in tree.css("a[href]"):
                href = a.attributes.get("href") or ""
                if href.startswith("/"):
                    href = f"https://{site}{href}"
                if not href.startswith("http") or site not in href:
                    continue
                cu = canonical_url(href)
                if cu in seen:
                    continue
                blob = f"{a.text(strip=True)} {urlparse(cu).path}".lower()
                score = sum(1 for t in terms if t in blob)
                if score:
                    seen.add(cu)
                    links.append((score, cu))
        links.sort(reverse=True)
        extra = [u for _s, u in links[: max_pages - len(docs)]]
        if extra:
            got = await gather_capped((take(u) for u in extra),
                                      limit=settings.scrape_concurrency)
            docs.extend([d for d in got if d])

    return docs