"""
intraday_score_study.py -- [2026-10-13] The user: intraday rescans must take the WHOLE universe and RE-SCORE every stock
with the information available AT THAT TIME (today the rescan re-uses the premarket score -> the list barely changes).
2 years (bars10y: every $5-15 stock with >= 500k shares that day), rescan times 10:00, 10:30, 11:00, 12:00, 13:00, 14:00.
Features at time t (bars up to t only): daily ATR %, |gap|, gap, move from the open (signed / absolute), range so far,
volume so far vs normal for that time, last-15-min volume vs normal, vs VWAP, position in the day range, 30-min trend
(signed straightness), below the day high.  "Normal volume" = the stock's 20-day average daily volume (days it was in
the universe) x the typical share of a day's volume traded by minute t (learned from the data, year 1).
Targets after t: UP-RUN = the high after t >= +3% over the price at t; MOVER = range after t >= 5%.
Score = rank average of the features picked on YEAR 1 (|AUC - 0.5| >= 0.04, with their signs); checked on YEAR 2.
Compared at each time: top 30 by the intraday score vs top 30 by the premarket scores (old v1 / v3, pm_history).
    python3 intraday_score_study.py          -> intraday_score_study.log / intraday_score_rows.npz / model json
"""
import gzip
import json
import sys
from collections import defaultdict
from pathlib import Path

import numpy as np

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
from up2_styles_build import arrays, cl  # noqa: E402

START, END, SPLIT = "2024-10-01", "2026-10-09", "2025-10-01"
TIMES = [30, 60, 90, 150, 210, 270]
F = ["atr", "absgap", "gap", "move", "absmove", "range", "rvol_t", "vol15_x", "vwap", "pos", "trend30", "off_hi"]
MODEL = Path("/var/www/screener/trade/data/intraday_score_model.json")


def pm_scores(day):
    f = HERE / "pm_history" / f"{day}.json.gz"
    if not f.exists():
        return {}, {}
    v1, v3 = {}, {}
    for r in json.load(gzip.open(f, "rt"))["rows"]:
        g = lambda k: r.get(k) if r.get(k) is not None else np.nan
        pc, atr, pmv, adv, gap, st, dist = g("prev_close"), g("atr"), g("pm_vol") or 0, g("adv20"), g("gap"), g("strength"), r.get("dist")
        gap = 0 if gap != gap else gap; st = 0 if st != st else st
        atrp = atr / pc * 100 if pc and atr == atr else 0; atrp = 0 if atrp != atrp or atrp > 25 else atrp
        ad = adv if adv and adv == adv else np.nan
        v1[r["s"]] = (30 * cl(pmv / ad / 3 if ad == ad else 0) + 20 * cl(pmv / 1e5) + 20 * cl(gap / 10, -1, 1) + 15 * cl(st / 5, -1, 1)
                      + 15 * (1 if dist is None else cl(dist / 8)))
        v3[r["s"]] = (35 * cl(atrp / 10) + 25 * cl(abs(gap) / 10) + 20 * cl(pmv / 1e5) + 10 * (cl(pmv / ad / 0.05) if ad == ad else 0)
                      + 10 * (1 if dist is None else cl(dist / 8)))
    return v1, v3


def build():
    from datetime import date, timedelta
    warm = (date.fromisoformat(START) - timedelta(days=60)).isoformat()      # only the 60 days before START for ATR / volume history
    files = [f for f in sorted((HERE / "bars10y").glob("*/*.json.gz")) if warm <= f.name[:10] <= END]
    hist = defaultdict(list)                         # sym -> [(h, l, c, v)] universe days
    rows = []; curve = np.zeros(390); ncurve = 0
    for f in files:
        day = f.name[:10]
        d = json.load(gzip.open(f, "rt")); daily = d.get("daily", {})
        live = day >= START
        if live:
            v1, v3 = pm_scores(day)
            for s, bars in d["bars"].items():
                hs = hist.get(s, [])
                if s not in daily or len(hs) < 6:
                    continue
                a = arrays(bars)
                if a is None:
                    continue
                O, H, L, C, V = a; O0 = float(O[0])
                pc = daily[s][5] if len(daily[s]) > 5 and daily[s][5] else hs[-1][2]
                tr = [max(b[0] - b[1], abs(b[0] - p[2]), abs(b[1] - p[2])) for p, b in zip(hs[-15:], hs[-14:])]
                atrp = np.mean(tr) / hs[-1][2] * 100
                adv = np.mean([x[3] for x in hs[-20:]])
                cv = np.cumsum(V); vw = np.cumsum((H + L + C) / 3 * V) / np.where(cv > 0, cv, np.nan)
                if day < SPLIT and cv[-1] > 0:
                    curve += cv / cv[-1]; ncurve += 1
                for t in TIMES:
                    k = t - 1
                    P = C[k]; hi = H[:t].max(); lo = L[:t].min()
                    path = np.abs(np.diff(C[k - 30:t])).sum()
                    up = (H[t:386].max() / P - 1) * 100; rng = (H[t:386].max() - L[t:386].min()) / P * 100
                    rows.append((day, s, t, atrp, abs(O0 / pc - 1) * 100, (O0 / pc - 1) * 100, (P / O0 - 1) * 100, abs(P / O0 - 1) * 100,
                                 (hi - lo) / O0 * 100, cv[k] / adv if adv else np.nan, V[t - 15:t].sum() / adv if adv else np.nan,
                                 (P / vw[k] - 1) * 100, (P - lo) / (hi - lo) if hi > lo else 0.5,
                                 (C[k] - C[k - 30]) / path if path > 0 else 0, (P / hi - 1) * 100,
                                 up >= 3, rng >= 5, v1.get(s, np.nan), v3.get(s, np.nan)))
        for s, x in daily.items():
            hist[s].append((x[1], x[2], x[3], x[4])); hist[s] = hist[s][-21:]
        del d
        if live:
            print(day, len(rows), flush=True)
    return rows, curve / max(ncurve, 1)


def auc(x, y):
    k = ~np.isnan(x); x, y = x[k], y[k]
    if y.sum() == 0 or (~y).sum() == 0:
        return np.nan
    r = np.argsort(np.argsort(x)) + 1.0
    return (r[y].sum() - y.sum() * (y.sum() + 1) / 2) / (y.sum() * (~y).sum())


def main():
    rows, curve = build()
    day = np.array([r[0] for r in rows]); t = np.array([r[2] for r in rows])
    X = {f: np.array([r[3 + i] for r in rows], float) for i, f in enumerate(F)}
    tmin = np.array([curve[max(0, x - 1)] for x in t])                       # share of a day's volume by time t
    X["rvol_t"] = X["rvol_t"] / np.where(tmin > 0, tmin, np.nan)
    tl15 = np.array([curve[max(0, x - 1)] - curve[max(0, x - 16)] for x in t])
    X["vol15_x"] = X["vol15_x"] / np.where(tl15 > 0, tl15, np.nan)
    upr = np.array([r[15] for r in rows]); mov = np.array([r[16] for r in rows])
    v1 = np.array([r[17] for r in rows], float); v3 = np.array([r[18] for r in rows], float)
    y1 = day < SPLIT
    print(f"\n{len(rows)} stock-time rows, {len(set(day))} days; UP-RUN (>= +3% after t) {upr.mean():.0%}, MOVER (range after >= 5%) {mov.mean():.0%}")
    print("\nAUC per feature (UP-RUN | MOVER), year 1 / year 2")
    pick = {}
    for f in F:
        a1, a2 = auc(X[f][y1], upr[y1]), auc(X[f][~y1], upr[~y1])
        m1, m2 = auc(X[f][y1], mov[y1]), auc(X[f][~y1], mov[~y1])
        print(f"  {f:9} up-run {a1:.2f} / {a2:.2f}   mover {m1:.2f} / {m2:.2f}")
        if abs(a1 - 0.5) >= 0.04:
            pick[f] = 1 if a1 > 0.5 else -1
    print("picked on year 1 (sign):", pick)
    # rank-average score within each (day, t) cross-section
    score = np.zeros(len(rows))
    key = defaultdict(list)
    for i, (d_, t_) in enumerate(zip(day, t)):
        key[(d_, t_)].append(i)
    for idx in key.values():
        idx = np.array(idx)
        for f, sgn in pick.items():
            x = X[f][idx] * sgn
            x = np.where(np.isnan(x), np.nanmin(x) if np.isfinite(np.nanmin(x)) else 0, x)
            score[idx] += np.argsort(np.argsort(x)) / max(len(idx) - 1, 1)
    print(f"\nscore AUC up-run: year 1 {auc(score[y1], upr[y1]):.2f} / year 2 {auc(score[~y1], upr[~y1]):.2f}   "
          f"mover {auc(score[y1], mov[y1]):.2f} / {auc(score[~y1], mov[~y1]):.2f}")
    print("\nTOP 30 at each rescan time: share that UP-RUN (+3% after) / MOVE (5% range after), year 1 | year 2")
    print(f"  {'time':6} {'intraday score':>24} {'old score (v1)':>24} {'v3 score':>24} {'all stocks':>16}")
    for tt in TIMES:
        res = {"new": [[], []], "v1": [[], []], "v3": [[], []]}
        for (d_, t_), idx in key.items():
            if t_ != tt:
                continue
            idx = np.array(idx); yi = 0 if d_ < SPLIT else 1
            for nm, sc in (("new", score), ("v1", v1), ("v3", v3)):
                s_ = np.where(np.isnan(sc[idx]), -1e9, sc[idx])
                top = idx[np.argsort(-s_)[:30]]
                res[nm][yi] += list(top)
        al = [y1 & (t == tt), ~y1 & (t == tt)]
        f = lambda L: f"{np.mean(upr[L]):.0%}/{np.mean(mov[L]):.0%}" if len(L) else "-"
        hm = f"{9 + (30 + tt) // 60}:{(30 + tt) % 60:02d}"
        print(f"  {hm:6} " + "".join(f"{f(res[nm][0]) + ' | ' + f(res[nm][1]):>24}" for nm in ("new", "v1", "v3"))
              + f"{f(np.where(al[0])[0]) + ' | ' + f(np.where(al[1])[0]):>16}")
    MODEL.parent.mkdir(parents=True, exist_ok=True)
    json.dump({"built": "2026-10-13", "features": pick, "volume_curve": [round(float(x), 5) for x in curve],
               "note": "rank-average of the picked features within each rescan's universe; see intraday_score_study.py"}, open(MODEL, "w"), indent=1)
    print("model ->", MODEL)


if __name__ == "__main__":
    main()
