← refacto-feature-scout

"""Reddit signal via public RSS/Atom feeds (the JSON API is blocked without OAuth; RSS is not)."""
from __future__ import annotations

import re
import time
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime, timedelta, timezone
from typing import Any, Callable

from .text import html_to_text

UA = "Mozilla/5.0 FeatureScout (+https://pypi.org/project/feature-scout/)"
ATOM = "{http://www.w3.org/2005/Atom}"
PAUSE_S = 12.0   # reddit 429s back-to-back feed requests; observed OK at ~10s+ spacing
RETRY_S = 45.0   # one retry after a 429

def fetch_feed(url: str) -> str:
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=30) as r:
        return r.read().decode("utf-8", "replace")


def parse_atom(xml_text: str) -> list[dict[str, Any]]:
    """Entries as {title, url, author, published, text}."""
    try:
        root = ET.fromstring(xml_text)
    except ET.ParseError:
        return []
    out = []
    for e in root.iter(ATOM + "entry"):
        title = (e.findtext(ATOM + "title") or "").strip()
        link = e.find(ATOM + "link")
        url = link.get("href", "") if link is not None else ""
        author = (e.findtext(ATOM + "author/" + ATOM + "name") or "").strip()
        published = (e.findtext(ATOM + "published") or e.findtext(ATOM + "updated") or "").strip()
        content = e.findtext(ATOM + "content") or ""
        text = re.sub(r"\s+", " ", html_to_text(content)).strip()
        text = re.sub(r"\[link\]|\[comments\]|submitted by\s+/u/\S+", "", text).strip()
        out.append({"title": title, "url": url, "author": author, "published": published, "text": text})
    return out


def _recent(entry: dict[str, Any], days: int) -> bool:
    if not entry.get("published"):
        return True
    try:
        ts = datetime.fromisoformat(entry["published"].replace("Z", "+00:00"))
    except ValueError:
        return True
    return ts >= datetime.now(timezone.utc) - timedelta(days=days)


def collect(urls: list[str], *, fetch: Callable[[str], str] = fetch_feed,
            days: int = 14, per_feed: int = 25, sleep: Callable[[float], None] = time.sleep) -> tuple[list[dict[str, Any]], list[str]]:
    """Recent posts from the given feeds, plus a list of feed-level problems (never raises).

    Takes the addresses directly: which communities matter is a thing each project decides,
    not something this file can know.
    """
    seen: set[str] = set()
    posts, problems = [], []
    for i, url in enumerate(urls):
        if i:
            sleep(PAUSE_S)
        try:
            entries = parse_atom(fetch(url))
        except urllib.error.HTTPError as e:
            if e.code != 429:
                problems.append(f"{url}: {e}")
                continue
            sleep(RETRY_S)
            try:
                entries = parse_atom(fetch(url))
            except Exception as e2:  # noqa: BLE001
                problems.append(f"{url}: {e2} (after retry)")
                continue
        except Exception as e:  # noqa: BLE001 — one dead feed must not sink the scan
            problems.append(f"{url}: {e}")
            continue
        if not entries:
            problems.append(f"{url}: no entries")
        for en in entries[:per_feed]:
            if en["url"] in seen or not _recent(en, days):
                continue
            seen.add(en["url"])
            posts.append(en)
    return posts, problems


def render(posts: list[dict[str, Any]], limit_chars: int = 9000) -> str:
    lines, used = [], 0
    for p in posts:
        line = f"- {p['title']} ({p['url']})"
        if p.get("text"):
            line += f": {p['text'][:280]}"
        if used + len(line) > limit_chars:
            break
        lines.append(line); used += len(line) + 1
    return "\n".join(lines) if lines else "(no Reddit posts fetched)"