"""
select_topn_test.py -- [2026-10-07] The user: bigger positions + fewer trades per day. First: does any SELECTION make the
average trade beat the real cost? 2-year v2 replay (live entry rules + live exits), scored at the REAL ~0.2% cost.
Causal selection (no peeking at later trades of the day): for each quality metric, the threshold is set on YEAR 1 so that
about N trades/day pass; then, in time order, the first trades of each day that pass are taken, at most N per day.
Metrics known at the buy: the new scanner score (v3, from pm_history), RVOL so far, buy share (15 min), distance above
VWAP (closer = better), move vs normal (smaller = better), smoothness R2, daily ATR %, gap size. Baselines: all trades;
the first N of the day.
    python3 select_topn_test.py
"""
import gzip
import json
from collections import defaultdict
from pathlib import Path

import numpy as np

HERE = Path("/var/www/screener/trade/reports")
T = [json.loads(l) for l in open(HERE / "playbook/v2_bar_replay_2024-10-01_2026-10-02.jsonl")]
for t in T:
    t["pl2"] = t["pl"] - 0.1                                   # 0.1% in the replay -> the real ~0.2%
# new scanner score per stock-day (pm_history = every universe stock, 2 years)
v3 = {}
for f in sorted((HERE / "open_window/pm_history").glob("20*.json.gz")):
    d = json.load(gzip.open(f, "rt"))
    for r in d["rows"]:
        try:
            atrp = r["atr"] / r["prev_close"] * 100 if r.get("atr") and r.get("prev_close") else 0
            atrp = 0 if atrp > 25 else atrp
            cl = lambda x: max(0.0, min(1.0, x))
            share = r["pm_vol"] / r["adv20"] if r.get("adv20") else 0
            v3[(d["date"], r["s"])] = (35 * cl(atrp / 10) + 25 * cl(abs(r["gap"]) / 10) + 20 * cl(r["pm_vol"] / 1e5)
                                       + 10 * cl(share / 0.05) + 10 * (1 if r.get("dist") is None else cl(r["dist"] / 8)))
        except Exception:
            pass
F = lambda t, k: t["feat"].get(k)
METRICS = {
    "new scanner score": (lambda t: v3.get((t["day"], t["sym"])), +1),
    "RVOL so far": (lambda t: F(t, "rvol_sofar"), +1),
    "buy share (15 min)": (lambda t: F(t, "buy_share15"), +1),
    "close to VWAP": (lambda t: F(t, "vwap_pct"), -1),
    "small move vs normal": (lambda t: F(t, "move_x"), -1),
    "smoothness R2": (lambda t: F(t, "r2_15"), +1),
    "daily ATR %": (lambda t: F(t, "atr_pct"), +1),
    "gap size": (lambda t: abs(F(t, "gap_pct") or 0), +1),
}
days = sorted({t["day"] for t in T}); y1 = [d for d in days if d < "2025-10-01"]
byday = defaultdict(list)
for t in sorted(T, key=lambda t: (t["day"], t["buy_m"])):
    byday[t["day"]].append(t)


def summary(sel, label):
    out = f"{label:44}"
    for lab, ds in (("year 1", [d for d in days if d < "2025-10-01"]), ("year 2", [d for d in days if d >= "2025-10-01"])):
        X = [t for d in ds for t in sel.get(d, [])]
        p = np.array([t["pl2"] for t in X]) if X else np.zeros(1)
        out += f" | {lab}: {len(X):4} trades, {p.mean():+.3f}%/trade, total {p.sum():+6.1f}%"
    print(out)


print(f"{len(T)} trades, {len(days)} days; per trade at the REAL ~0.2% cost (break-even = 0.000%)\n")
summary({d: v for d, v in byday.items()}, "ALL trades")
for N in (1, 2, 3, 5):
    print(f"\n--- at most {N} trade(s) per day ---")
    summary({d: v[:N] for d, v in byday.items()}, f"first {N} of the day")
    for name, (fn, sgn) in METRICS.items():
        vals = [(fn(t), t) for d in y1 for t in byday[d] if fn(t) is not None]
        if len(vals) < 50:
            continue
        x = np.array([v for v, _ in vals]) * sgn
        k = min(len(x) - 1, int(N * len(y1)))                  # threshold so that ~N/day pass on year 1
        thr = np.sort(x)[::-1][k]
        sel = {}
        for d, v in byday.items():
            ok = [t for t in v if fn(t) is not None and fn(t) * sgn >= thr][:N]
            sel[d] = ok
        summary(sel, f"best by {name} (thr {thr * sgn:.2f})")
