"""
news_test.py -- [2026-10-12] The user: does NEWS (Benzinga via Alpaca, trade/news_recorder.py tags) predict which scanner
picks run, and does it help or hurt the bot's buys? 2 years (Oct 2024 - Oct 2026).
  backfill   python3 news_test.py --backfill 2024-10-01..2026-10-09   (data/news/<day>.jsonl for every trading day)
  test       python3 news_test.py
PICKS  = the daily top-30 lists (reports/open_window/bars). News known AT THE OPEN = from the previous trading day 16:00
         to 9:30. Outcome of the stock-day: reached +3% / +5% over the open, close vs open.
TRADES = the 'BASE' trades of avoid_rules_test.jsonl (live v2 buys + live exits, every signal), at the real 0.2% cost.
         News known AT THE BUY = previous day 16:00 until the buy minute.
Per tag and "any news" vs "no news", year 1 (< 2025-10-01) vs year 2.
"""
import gzip
import json
import sys
import time
from collections import defaultdict
from datetime import datetime, timedelta
from pathlib import Path
from zoneinfo import ZoneInfo

import numpy as np

HERE = Path(__file__).resolve().parent
TRADE = Path("/var/www/screener/trade")
OW = TRADE / "reports" / "open_window"
sys.path.insert(0, str(TRADE))
ET = ZoneInfo("America/New_York")
COST, SPLIT = 0.2, "2025-10-01"


def days_list():
    return sorted(p.name[5:15] for p in OW.glob("bars/bars_20*.json.gz"))


def backfill(rng):
    import news_recorder as NR
    from config_loader import get_env
    e = get_env(); h = {"APCA-API-KEY-ID": e.api_key, "APCA-API-SECRET-KEY": e.secret_key}
    a, b = rng.split("..")
    for day in [d for d in days_list() if a <= d <= b]:
        p = TRADE / "data" / "news" / f"{day}.jsonl"
        if p.exists() and p.stat().st_size > 1000:
            continue
        d0 = datetime.fromisoformat(day).replace(tzinfo=ET)
        for attempt in range(3):
            try:
                n = NR.pull(h, d0, d0 + timedelta(days=1), NR.load_seen(day)); break
            except Exception as ex:
                print(day, "retry", ex, flush=True); time.sleep(10); n = 0
        print(day, n, "articles", flush=True)
        time.sleep(0.5)
    print("NEWS_BACKFILL_DONE")


def news_index():
    """sym -> sorted [(datetime ET, tags)]"""
    idx = defaultdict(list)
    for p in sorted((TRADE / "data" / "news").glob("20*.jsonl")):
        for l in open(p):
            r = json.loads(l)
            t = datetime.fromisoformat(r["time"]).replace(tzinfo=ET)
            for s in r["symbols"]:
                idx[s].append((t, r["tags"] or ["untagged"]))
    for s in idx:
        idx[s].sort(key=lambda x: x[0])
    return idx


def window(idx, sym, start, end):
    tags = set(); n = 0
    for t, tg in idx.get(sym, ()):
        if start <= t < end:
            n += 1; tags |= set(tg)
    return n, tags


def main():
    idx = news_index()
    days = days_list()
    prev = {d: days[i - 1] for i, d in enumerate(days) if i}
    print(f"news index: {sum(len(v) for v in idx.values())} symbol-articles, {len(idx)} symbols")
    # ---- PICKS
    P = []
    for d in days:
        if d not in prev:
            continue
        dd = json.load(gzip.open(OW / "bars" / f"bars_{d}.json.gz", "rt"))
        st = datetime.fromisoformat(prev[d]).replace(hour=16, tzinfo=ET); op = datetime.fromisoformat(d).replace(hour=9, minute=30, tzinfo=ET)
        for s in dd["top30_current"]:
            b = (dd["symbols"].get(s) or {}).get("bars") or []
            if len(b) < 60:
                continue
            O, H, C = b[0][1], max(x[2] for x in b), b[-1][4]
            n, tags = window(idx, s, st, op)
            P.append({"day": d, "y2": d >= SPLIT, "n": n, "tags": tags, "up3": H >= O * 1.03, "up5": H >= O * 1.05, "close": (C / O - 1) * 100})
        del dd
    def pick_line(lab, X):
        if not X:
            return f"  {lab:28} 0"
        out = []
        for nm, sel in (("y1", lambda r: not r["y2"]), ("y2", lambda r: r["y2"])):
            Z = [r for r in X if sel(r)]
            out.append(f"{nm} {len(Z):5} | +3% {np.mean([r['up3'] for r in Z]) if Z else 0:4.0%} +5% {np.mean([r['up5'] for r in Z]) if Z else 0:4.0%} "
                       f"close {np.mean([r['close'] for r in Z]) if Z else 0:+.2f}%")
        return f"  {lab:28} " + " || ".join(out)
    print(f"\nPICKS (top-30 stock-days: {len(P)}): news known at the open (prev 16:00 -> 9:30)")
    print(pick_line("no news", [r for r in P if not r["n"]]))
    print(pick_line("ANY news", [r for r in P if r["n"]]))
    alltags = sorted({t for r in P for t in r["tags"]})
    for t in alltags:
        X = [r for r in P if t in r["tags"]]
        if len(X) >= 40:
            print(pick_line(t, X))
    # ---- TRADES
    tf = HERE / "avoid_rules_test.jsonl"
    if not tf.exists():
        print("\n(no avoid_rules_test.jsonl yet -- trade part skipped)"); return
    T = [json.loads(l) for l in open(tf)]
    T = [t for t in T if t["run"] == "BASE"]
    for t in T:
        st = datetime.fromisoformat(prev.get(t["day"], t["day"])).replace(hour=16, tzinfo=ET)
        bt = datetime.fromisoformat(t["day"]).replace(hour=9, minute=30, tzinfo=ET) + timedelta(minutes=t["buy_m"])
        t["n"], t["tags"] = window(idx, t["sym"], st, bt)
        t["pl"] = t["pl_gross"] - COST
    def trade_line(lab, X):
        out = []
        for nm, sel in (("y1", lambda r: r["day"] < SPLIT), ("y2", lambda r: r["day"] >= SPLIT)):
            Z = np.array([r["pl"] for r in X if sel(r)])
            out.append(f"{nm} {len(Z):5} {Z.mean() if len(Z) else 0:+.3f}%/trade win {np.mean(Z > 0) if len(Z) else 0:.0%}")
        return f"  {lab:28} " + " || ".join(out)
    print(f"\nTRADES (live v2 buys, every signal: {len(T)}): news known at the buy (prev 16:00 -> buy minute), {COST}% cost")
    print(trade_line("no news", [t for t in T if not t["n"]]))
    print(trade_line("ANY news", [t for t in T if t["n"]]))
    for tg in sorted({x for t in T for x in t["tags"]}):
        X = [t for t in T if tg in t["tags"]]
        if len(X) >= 25:
            print(trade_line(tg, X))
    print("NEWS_TEST_DONE")


if __name__ == "__main__":
    if "--backfill" in sys.argv:
        backfill(sys.argv[sys.argv.index("--backfill") + 1])
    else:
        main()
