"""
weekly_review.py -- [2026-10-11] The user: a WEEKLY review of the bots and of everything we only RECORD.
Every Friday after the recorders ran (cron 18:10) -> reports/weekly/<friday>_weekly_review.html (open in the browser:
http://<server>/trade/reports/weekly/). Read only; nothing live is touched.
Sections (this week, and since the start of each recorder):
  1  both bots, REAL broker fills: trades, win %, P/L per day          2  scanner A/B: trade bot (v3) vs Star bot (old
     score) -- how many picks rose >= 3% / >= 5% over the open (bars10y daily)
  3  bot buys, Star sells (star_exit_daily.jsonl) incl. the noise2 filter
  4  deep-V recorder (deepv_daily.jsonl, 0.2% cost)    5  group watcher + its comparison with the bots' buys
  6  SEC filings on watched / traded stocks             7  order book (data/book): bid/ask imbalance before the bots' buys
    python3 weekly_review.py [FRIDAY]        default: the last Friday (today if Friday)
"""
import glob
import gzip
import html
import json
import os
import sys
from collections import Counter, defaultdict
from datetime import date, datetime, timedelta
from pathlib import Path
from zoneinfo import ZoneInfo

TRADE = Path("/var/www/screener/trade"); STAR = Path("/var/www/screener/premarket")
OUT = TRADE / "reports" / "weekly"
ET = ZoneInfo("America/New_York")
COST = 0.2


def jl(p):
    return [json.loads(l) for l in open(p)] if os.path.exists(p) else []


def table(head, rows):
    h = "".join(f"<th>{html.escape(str(x))}</th>" for x in head)
    b = "".join("<tr>" + "".join(f"<td>{html.escape(str(x))}</td>" for x in r) + "</tr>" for r in rows)
    return f"<table><tr>{h}</tr>{b}</table>"


def money(x):
    return f"${x:+,.2f}"


def bots(days):
    out = []
    rows = []
    tot = {}
    for name, root in (("trade bot (v3 scanner)", TRADE), ("Star bot (old scanner)", STAR)):
        n = w = 0; pl = 0.0
        for d in days:
            T = [t for t in jl(root / "data" / "trades" / f"{d}_trades.jsonl") if t.get("exit_price")]
            p = sum(t.get("pl_dollars") or 0 for t in T); wins = sum((t.get("pl_dollars") or 0) > 0 for t in T)
            rec = sum(1 for t in T if t.get("fills_reconciled"))
            rows.append([d, name, len(T), f"{wins}/{len(T)}", money(p), f"{rec}/{len(T)}"])
            n += len(T); w += wins; pl += p
        tot[name] = (n, w, pl)
    out.append(table(["day", "bot", "trades", "wins", "P/L (real fills)", "reconciled"], rows))
    out.append(table(["bot", "trades", "win %", "P/L week", "per trade"],
                     [[k, n, f"{w / n:.0%}" if n else "-", money(p), money(p / n) if n else "-"] for k, (n, w, p) in tot.items()]))
    # since v2 went live
    allrows = []
    for name, root, start in (("trade bot", TRADE, "2026-09-29"), ("Star bot", STAR, "2026-10-05")):
        T = []
        for f in sorted(glob.glob(str(root / "data" / "trades" / "2026-*_trades.jsonl"))):
            if os.path.basename(f)[:10] >= start:
                T += [t for t in jl(f) if t.get("exit_price")]
        p = sum(t.get("pl_dollars") or 0 for t in T)
        allrows.append([name, f"since {start}", len(T), f"{sum((t.get('pl_dollars') or 0) > 0 for t in T) / max(len(T), 1):.0%}", money(p),
                        money(p / max(len(T), 1))])
    out.append("<p>Since the start:</p>" + table(["bot", "period", "trades", "win %", "P/L", "per trade"], allrows))
    return "".join(out)


def scanner_ab(days):
    rows = []
    for d in days:
        f = TRADE / "reports" / "open_window" / "bars10y" / d[:4] / f"{d}.json.gz"
        if not f.exists():
            rows.append([d, "no daily bars yet", "", "", ""]); continue
        daily = json.load(gzip.open(f, "rt")).get("daily", {})
        for name, root in (("trade bot (v3)", TRADE), ("Star bot (old)", STAR)):
            p = root / "data" / "candidates" / f"{d}_scanner.json"
            if not p.exists():
                continue
            syms = [c["symbol"] for c in json.load(open(p))]
            g = [(daily[s][1] / daily[s][0] - 1) * 100 for s in syms if s in daily and daily[s][0]]
            rows.append([d, name, f"{len(syms)} ({len(g)} with bars)", f"{sum(x >= 3 for x in g)} ({sum(x >= 3 for x in g) / max(len(g), 1):.0%})",
                         f"{sum(x >= 5 for x in g)} ({sum(x >= 5 for x in g) / max(len(g), 1):.0%})"])
    return ("<p>The 9:30 list of each bot; how many picks rose >= 3% / >= 5% over the open during the day.</p>"
            + table(["day", "bot", "picks", ">= +3% over open", ">= +5% over open"], rows))


def star_exit(days):
    R = jl(TRADE / "data" / "playbook" / "star_exit_daily.jsonl")
    out = []
    for lab, sel in (("this week", lambda r: r["day"] in days), ("since the start", lambda r: True)):
        rows = []
        for src in ("trade_bot", "star_bot"):
            X = [r for r in R if sel(r) and r.get("star_pl") is not None and r.get("source", "trade_bot") == src]
            if not X:
                continue
            n2 = [r for r in X if r.get("n2_pl") is not None]
            rows.append([src, len(X), f"{sum(r['bot_pl'] for r in X):+.2f}%", f"{sum(r['star_pl'] for r in X):+.2f}%",
                         f"{sum(r['star_pl'] > r['bot_pl'] for r in X)}/{len(X)}",
                         f"{sum(r['n2_pl'] for r in n2):+.2f}% vs Star {sum(r['star_pl'] for r in n2):+.2f}% ({len(n2)})" if n2 else "-"])
        out.append(f"<p><b>{lab}</b> (sum of % per trade, 0.1% cost each side as recorded)</p>" + table(
            ["buys of", "trades", "bot's own exits", "Star's sells", "Star better", "Star + noise2"], rows))
    return "".join(out)


def deepv(days):
    R = jl(TRADE / "data" / "playbook" / "deepv_daily.jsonl")
    groups = [("drop 2%", lambda r: r["drop"] == 2), ("drop 3%", lambda r: r["drop"] == 3), ("drop 5%", lambda r: r["drop"] == 5),
              ("drop 5% + dried", lambda r: r["drop"] == 5 and r["dried"]),
              ("drop 5% + dried, in a bot scan", lambda r: r["drop"] == 5 and r["dried"] and (r["in_trade_scan"] or r["in_star_scan"])),
              ("drop 3% + dried, in a bot scan", lambda r: r["drop"] == 3 and r["dried"] and (r["in_trade_scan"] or r["in_star_scan"]))]
    rows = []
    for lab, sel in groups:
        for per, ps in (("week", lambda r: r["day"] in days), ("all", lambda r: True)):
            X = [r for r in R if sel(r) and ps(r)]
            if X:
                rows.append([lab, per, len(X), f"{sum(r['r1'] for r in X) / len(X):.0%}",
                             f"{sum(r['pl2R_gross'] - COST for r in X) / len(X):+.3f}%", f"{sum(r['hold_gross'] - COST for r in X) / len(X):+.3f}%"])
    return f"<p>First causal deep V per stock, buy at the turn, stop under the V low; {COST}% cost.</p>" + table(
        ["group", "period", "trades", "+1R reached", "2R trade", "hold to close"], rows)


def groups(days):
    rows = []
    for d in days:
        G = jl(TRADE / "data" / "group_watch" / f"{d}.jsonl")
        if G:
            c = Counter(r["label"] for r in G)
            rows.append([d, len(G), sum(r.get("signal") == "BUY" for r in G), ", ".join(f"{k} {v}" for k, v in c.most_common())])
    C = [r for r in jl(TRADE / "data" / "group_watch" / "compare.jsonl")]
    by = defaultdict(lambda: [0, 0.0])
    for r in C:
        if r["kind"] == "bot_entry":
            k = r.get("label_before") or "not grouped"
            by[k][0] += 1; by[k][1] += r.get("pl") or 0
    sig = [r for r in C if r["kind"] == "signal"]
    out = table(["day", "group checks", "buy signals", "labels"], rows)
    out += "<p>The bots' real buys by the group the stock was in just before (since the start):</p>" + table(
        ["group label before the buy", "buys", "P/L"], [[k, n, money(p)] for k, (n, p) in sorted(by.items())])
    if sig:
        out += "<p>Group buy signals:</p>" + table(["day", "stock", "time", "price", "result", "P/L %", "max after %"],
                                                    [[r["day"], r["sym"], r["sig_t"], r["sig_price"], r["res"], r["pl"], r["max_after"]] for r in sig])
    return out


def edgar(days):
    rows = []; flagged = []
    for d in days:
        ev = jl(TRADE / "data" / "events" / f"{d}.jsonl")
        if not ev:
            continue
        cands, tr = set(), {}
        for root in (TRADE, STAR):
            for f in glob.glob(str(root / "data" / "candidates" / f"{d}_scanner*.json")):
                cands |= {c["symbol"] for c in json.load(open(f))}
            for t in jl(root / "data" / "trades" / f"{d}_trades.jsonl"):
                tr[t["symbol"]] = tr.get(t["symbol"], 0) + (t.get("pl_dollars") or 0)
        hit = [(t, e["verdict"], e["tags"][0][0] if e["tags"] else e["form"]) for e in ev for t in e["tickers"] if t in cands]
        c = Counter(v for _, v, _ in hit)
        rows.append([d, len(ev), len(hit), len({h[0] for h in hit}), ", ".join(f"{k} {v}" for k, v in c.most_common())])
        for t, v, tg in hit:
            if v in ("avoid", "catalyst") or t in tr:
                flagged.append([d, t, v, tg, money(tr[t]) if t in tr else "not traded"])
    out = table(["day", "filings", "on watched stocks", "stocks", "verdicts"], rows)
    if flagged:
        out += "<p>Avoid / catalyst filings on watched stocks, and every filing on a traded stock:</p>" + table(
            ["day", "stock", "verdict", "tag", "bots' P/L that day"], sorted({tuple(x) for x in flagged}))
    return out


def news(days):
    """[2026-10-12] Benzinga headlines (news_recorder.py) on the bots' watched / traded stocks."""
    rows, traded_rows = [], []
    for d in days:
        N = jl(TRADE / "data" / "news" / f"{d}.jsonl")
        if not N:
            continue
        cands, tr = set(), {}
        for root in (TRADE, STAR):
            for f in glob.glob(str(root / "data" / "candidates" / f"{d}_scanner*.json")):
                cands |= {c["symbol"] for c in json.load(open(f))}
            for t in jl(root / "data" / "trades" / f"{d}_trades.jsonl"):
                tr[t["symbol"]] = tr.get(t["symbol"], 0) + (t.get("pl_dollars") or 0)
        w = [n for n in N if set(n["symbols"]) & cands]
        tags = Counter(t for n in w for t in n["tags"])
        rows.append([d, len(N), len(w), len({s for n in w for s in n["symbols"] if s in cands}), ", ".join(f"{k} {v}" for k, v in tags.most_common(6))])
        for s, pl in tr.items():
            hs = [n for n in N if s in n["symbols"]]
            traded_rows.append([d, s, money(pl), len(hs), "; ".join(f"{n['time'][11:16]} {n['headline'][:70]}" for n in hs[:3]) or "no news"])
    if not rows:
        return "<p>No news recorded yet (news_recorder.py starts 2026-10-12).</p>"
    return (table(["day", "articles (all)", "on watched stocks", "watched stocks with news", "top tags"], rows)
            + "<p>Traded stocks and their news that day:</p>" + table(["day", "stock", "bots' P/L", "articles", "first headlines"], traded_rows))


def book(days):
    rows = []; entries = []
    for d in days:
        B = jl(TRADE / "data" / "book" / f"{d}.jsonl")
        if not B:
            continue
        rows.append([d, len(B), len({r["sym"] for r in B})])
        by = defaultdict(dict)
        for r in B:
            by[r["sym"]][r["minute"]] = r
        for root, name in ((TRADE, "trade bot"), (STAR, "Star bot")):
            for t in jl(root / "data" / "trades" / f"{d}_trades.jsonl"):
                if not t.get("entry_time"):
                    continue
                et = datetime.fromisoformat(t["entry_time"]).astimezone(ET)
                mins = [(et - timedelta(minutes=k)).strftime("%H:%M") for k in range(1, 6)]
                im = [by[t["symbol"]][m]["imbalance"] for m in mins if m in by.get(t["symbol"], {}) and by[t["symbol"]][m]["imbalance"] is not None]
                entries.append((name, t["symbol"], et.strftime("%H:%M"), sum(im) / len(im) if im else None, t.get("pl_dollars") or 0))
    if not rows:
        return "<p>No order-book data yet (the recorder starts 2026-10-12, trade bot's watch list).</p>"
    out = table(["day", "symbol-minutes", "symbols"], rows)
    if entries:
        w = [e for e in entries if e[3] is not None and e[4] > 0]; l = [e for e in entries if e[3] is not None and e[4] <= 0]
        avg = lambda X: f"{sum(e[3] for e in X) / len(X):+.3f}" if X else "-"
        out += (f"<p>Bid/ask size imbalance in the 5 minutes before each bot buy (+1 = all bids, -1 = all asks): "
                f"winners {avg(w)} (n {len(w)}), losers {avg(l)} (n {len(l)}).</p>")
        out += table(["bot", "stock", "buy time", "imbalance 5 min before", "P/L"],
                     [[a, b, c, f"{d:+.3f}" if d is not None else "-", money(e)] for a, b, c, d, e in entries])
    return out


def main(friday=None):
    today = date.today()
    fri = date.fromisoformat(friday) if friday else today - timedelta(days=(today.weekday() - 4) % 7)
    days = [(fri - timedelta(days=k)).isoformat() for k in range(4, -1, -1)]
    sections = [("1. Both bots -- real broker fills", bots), ("2. Scanner A/B -- quality of the 9:30 lists", scanner_ab),
                ("3. Bot buys, Star sells (daily check, record only)", star_exit), ("4. Deep-V recorder (record only)", deepv),
                ("5. Group watcher (record only, trade bot)", groups), ("6. SEC filings recorder (record only)", edgar),
                ("7. Order book recorder (record only, trade bot)", book),
                ("8. Benzinga news recorder (record only)", news)]
    body = []
    for title, fn in sections:
        try:
            body.append(f"<h2>{title}</h2>" + fn(days))
        except Exception as e:
            body.append(f"<h2>{title}</h2><p>could not build: {html.escape(repr(e))}</p>")
    page = f"""<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
<title>Weekly review {fri}</title><style>
body{{font-family:system-ui,Arial,sans-serif;margin:16px;color:#1d2733;background:#fff;max-width:1100px}}
h1{{font-size:22px}} h2{{font-size:17px;margin-top:28px;border-bottom:1px solid #d5dbe3;padding-bottom:4px}}
table{{border-collapse:collapse;margin:8px 0;font-size:13px;display:block;overflow-x:auto}}
th,td{{border:1px solid #d5dbe3;padding:4px 8px;text-align:right;white-space:nowrap}} th{{background:#f1f4f8}}
td:first-child,th:first-child{{text-align:left}} p{{font-size:14px}}
</style></head><body><h1>Weekly review -- week of {days[0]} to {days[-1]}</h1>
<p>Built {datetime.now(ET):%Y-%m-%d %H:%M} ET. Record-only items never change a trade. Small samples: judge over several weeks.</p>
{''.join(body)}</body></html>"""
    OUT.mkdir(parents=True, exist_ok=True)
    p = OUT / f"{fri}_weekly_review.html"
    p.write_text(page)
    print("->", p)


if __name__ == "__main__":
    main(sys.argv[1] if len(sys.argv) > 1 else None)
