"""
news_recorder.py -- [2026-10-12] RECORD ONLY (the user): every Benzinga headline (Alpaca news API, v1beta1/news) for ALL
stocks, polled every 60 s from 6:00 to 20:00 ET on weekdays, so we can later test whether news predicts which scanner
picks run. Nothing live reads it.
Output: data/news/<date>.jsonl, one line per article (first time seen): id, time (ET), headline, summary, symbols,
source, author, url, tags. Tags are simple keyword groups (a starting point, to be tested):
  analyst_up / analyst_down / analyst_init / analyst_other, price_target_up / price_target_down, earnings, guidance,
  why_moving, halt, offering, fda, merger, contract, insider, legal, management, buyback, dividend
Logs: logs/news_recorder.log. cron 6:00 weekdays (flock); it exits at 20:00.
    python3 news_recorder.py            python3 news_recorder.py --backfill 2026-10-12   (one past day)
"""
import json
import re
import sys
import time
from datetime import datetime, timedelta
from pathlib import Path
from zoneinfo import ZoneInfo

import requests

BASE = Path(__file__).resolve().parent
sys.path.insert(0, str(BASE))
from config_loader import get_env  # noqa: E402

ET = ZoneInfo("America/New_York")
URL = "https://data.alpaca.markets/v1beta1/news"
TAGS = [
    ("analyst_up", r"\bupgrade[sd]?\b|\braises? .*to (buy|outperform|overweight)"),
    ("analyst_down", r"\bdowngrade[sd]?\b"),
    ("analyst_init", r"\binitiates? coverage\b|\bbegins coverage\b|\bstarts coverage\b"),
    ("analyst_other", r"\bmaintains\b|\breiterates?\b"),
    ("price_target_up", r"raises price target"),
    ("price_target_down", r"lowers price target"),
    ("earnings", r"\bearnings\b|\bq[1-4] (20\d\d )?results\b|\beps\b|\brevenue\b|\btranscript\b"),
    ("guidance", r"\bguidance\b|\boutlook\b|\bforecast\b"),
    ("why_moving", r"what'?s going on|why is .* (stock|shares)|shares are trading (higher|lower)|stocks moving|movers"),
    ("halt", r"\bhalted\b|\bcircuit breaker\b"),
    ("offering", r"\boffering\b|\bprivate placement\b|\bregistered direct\b|\bdilut"),
    ("fda", r"\bfda\b|\bphase [123i]+\b|\bclinical\b|\btrial\b"),
    ("merger", r"\bacquir|\bmerger\b|\bto be acquired\b|\bbuyout\b|\btakeover\b"),
    ("contract", r"\bcontract\b|\bpartnership\b|\bpartners with\b|\bagreement\b|\bawarded\b|\bdeal\b"),
    ("insider", r"\binsider\b|\bform 4\b|\bbuys .* shares\b"),
    ("legal", r"\blawsuit\b|\bcourt\b|\bsec (charges|probe)|\binvestigation\b|\bsettlement\b|\brestraining order\b"),
    ("management", r"\bceo\b|\bcfo\b|\bappoints?\b|\bresigns?\b|\bsteps down\b"),
    ("buyback", r"\bbuyback\b|\brepurchase\b"),
    ("dividend", r"\bdividend\b"),
]
TAG_RE = [(k, re.compile(p, re.I)) for k, p in TAGS]


def tag(text):
    return [k for k, rx in TAG_RE if rx.search(text or "")]


def fetch(h, start, end, token=None):
    p = {"start": start.isoformat(), "end": end.isoformat(), "limit": 50, "sort": "asc", "include_content": "false"}
    if token:
        p["page_token"] = token
    r = requests.get(URL, headers=h, params=p, timeout=20)
    r.raise_for_status()
    j = r.json()
    return j.get("news", []), j.get("next_page_token")


def save(arts, seen):
    n = 0
    for a in arts:
        if a["id"] in seen:
            continue
        seen.add(a["id"])
        t = datetime.fromisoformat(a["created_at"].replace("Z", "+00:00")).astimezone(ET)
        row = {"id": a["id"], "time": t.strftime("%Y-%m-%d %H:%M:%S"), "headline": a.get("headline"),
               "summary": (a.get("summary") or "")[:500], "symbols": a.get("symbols") or [], "source": a.get("source"),
               "author": a.get("author"), "url": a.get("url"), "tags": tag((a.get("headline") or "") + " " + (a.get("summary") or ""))}
        p = BASE / "data" / "news" / f"{t.date().isoformat()}.jsonl"
        p.parent.mkdir(parents=True, exist_ok=True)
        with open(p, "a") as fh:
            fh.write(json.dumps(row) + "\n")
        n += 1
    return n


def load_seen(day):
    p = BASE / "data" / "news" / f"{day}.jsonl"
    return {json.loads(l)["id"] for l in open(p)} if p.exists() else set()


def pull(h, start, end, seen):
    n, token = 0, None
    while True:
        arts, token = fetch(h, start, end, token)
        n += save(arts, seen)
        if not token:
            return n


def main():
    e = get_env(); h = {"APCA-API-KEY-ID": e.api_key, "APCA-API-SECRET-KEY": e.secret_key}
    if "--backfill" in sys.argv:
        day = sys.argv[sys.argv.index("--backfill") + 1]
        d0 = datetime.fromisoformat(day).replace(tzinfo=ET)
        seen = load_seen(day)
        print(day, pull(h, d0, d0 + timedelta(days=1), seen), "articles saved", flush=True)
        return
    now = datetime.now(ET)
    seen = load_seen(now.date().isoformat())
    since = now.replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(hours=8)   # catch the overnight news too
    print(f"[{now:%H:%M:%S}] news recorder start", flush=True)
    while True:
        now = datetime.now(ET)
        if now.hour >= 20:
            break
        try:
            n = pull(h, since - timedelta(minutes=10), now, seen)   # overlap: late-indexed articles; ids dedupe
            if n:
                print(f"[{now:%H:%M:%S}] +{n} articles", flush=True)
            since = now
        except Exception as ex:
            print(f"[{now:%H:%M:%S}] error {ex}", flush=True)
        time.sleep(60)
    print(f"[{datetime.now(ET):%H:%M:%S}] stop", flush=True)


if __name__ == "__main__":
    main()
