"""
star_lists.py -- [2026-10-24] the user: connect EV3 to the STAR bot's stock selection and simulate the last 2 years.
Star's scanner (premarket/config.json): score_mode v1 (old score: 30 rvol + 20 pm volume + 20 gap + 15 pm strength + 15 room),
top 50 at 9:30; from 9:50 (no_rescan_before 20 min) every 5 min the WHOLE universe re-scored with the intraday model
(premarket/scanner.intraday_features + apply_intraday_score, data/intraday_score_model.json), top 50.
Universe = the stocks with pm_history rows AND bars10y minute bars that day (bars10y = opened $5-15 with >= 500k shares).
Output per day: reports/ev3/star_lists/<day>.json {"pre": [50], "rescans": {minute: [50]}, "ref": {sym: {...}}, "union": [...]}
    python3 star_lists.py START END
"""
import gzip, json, math, sys
from pathlib import Path
import numpy as np
OW = Path("/var/www/screener/trade/reports/open_window"); HERE = Path(__file__).resolve().parent
OUT = HERE / "star_lists"; OUT.mkdir(exist_ok=True)
MODEL = json.load(open("/var/www/screener/premarket/data/intraday_score_model.json"))
CURVE = MODEL["volume_curve"]
cl = lambda x, lo=0.0, hi=1.0: min(max(x, lo), hi)


def v1(r):
    pmv = r.get("pm_vol") or 0; adv = r.get("adv20") or 0; d = r.get("dist")
    return (30 * (cl(pmv / adv / 3) if adv else 0) + 20 * cl(pmv / 1e5) + 20 * cl((r.get("gap") or 0) / 10, -1, 1)
            + 15 * cl((r.get("strength") or 0) / 5, -1, 1) + 15 * (1 if d is None else cl(d / 8)))


def minute(ts):
    from datetime import datetime
    from zoneinfo import ZoneInfo
    t = datetime.fromtimestamp(ts, ZoneInfo("America/New_York")); return (t.hour - 9) * 60 + t.minute - 30


def day_lists(day, prev_rows):
    pm = json.load(gzip.open(OW / "pm_history" / f"{day}.json.gz", "rt")); rows = {r["s"]: r for r in pm["rows"]}
    bars = json.load(gzip.open(OW / "bars10y" / day[:4] / f"{day}.json.gz", "rt"))["bars"]
    uni = [s for s in rows if s in bars and rows[s].get("prev_close")]
    pre = sorted(uni, key=lambda s: v1(rows[s]), reverse=True)[:50]
    # per stock minute arrays
    A = {}
    for s in uni:
        o = np.full(390, np.nan); h = np.full(390, np.nan); l = np.full(390, np.nan); c = np.full(390, np.nan); v = np.zeros(390)
        for b in bars[s]:
            k = minute(b[0])
            if 0 <= k < 390:
                o[k], h[k], l[k], c[k], v[k] = b[1], b[2], b[3], b[4], b[5]
        A[s] = (o, h, l, c, v)
    rescans = {}
    for t in range(20, 390, 5):                 # rescan at minute t uses the bars of minutes < t
        feats = []
        for s in uni:
            o, h, l, c, v = A[s]; ok = ~np.isnan(c[:t])
            if ok.sum() < 5:
                continue
            idx = np.where(ok)[0]; O = o[idx[0]]; P = c[idx[-1]]; hi = np.nanmax(h[:t]); lo = np.nanmin(l[:t])
            r = rows[s]; pc = r["prev_close"]; adv = r.get("adv20")
            cv = v[:t].sum(); v15 = v[idx[-15:]].sum()
            tt = max(1, min(t, 389)); share = CURVE[tt - 1]; share15 = CURVE[tt - 1] - CURVE[max(0, tt - 16)]
            f = {"atr": (r.get("atr") or 0) / pc * 100 if r.get("atr") else None, "absgap": abs(O / pc - 1) * 100,
                 "absmove": abs(P / O - 1) * 100, "range": (hi - lo) / O * 100,
                 "rvol_t": cv / (adv * share) if adv and share > 0 else None,
                 "vol15_x": v15 / (adv * share15) if adv and share15 > 0 else None, "off_hi": (P / hi - 1) * 100}
            feats.append((s, f))
        if len(feats) < 10:
            continue
        tot = {s: 0.0 for s, _ in feats}
        for feat, sign in MODEL["features"].items():
            known = sorted((f[feat] * sign, s) for s, f in feats if f.get(feat) is not None and not math.isnan(f[feat]))
            for i, (_, s) in enumerate(known):
                tot[s] += i / max(len(known) - 1, 1)
        rescans[t] = [s for s, _ in sorted(tot.items(), key=lambda z: -z[1])[:50]]
    union = sorted(set(pre) | {s for L in rescans.values() for s in L})
    ref = {}
    for s in union:
        r = rows[s]; vh = None
        ref[s] = {"atr14": r.get("atr"), "prev_close": r.get("prev_close"), "prev_high": r.get("prev_high"),
                  "prev_low": (prev_rows.get(s) or {}).get("l")}
    (OUT / f"{day}.json").write_text(json.dumps({"date": day, "pre": pre, "rescans": rescans, "ref": ref, "union": union}))
    return rows, len(union)


def main():
    a, z = sys.argv[1], sys.argv[2]; prev_rows = {}
    files = sorted((OW / "pm_history").glob("20*.json.gz"))
    for i, f in enumerate(files):
        day = f.name[:10]
        if day > z:
            break
        if day < a:
            continue
        if i and not prev_rows:
            prev_rows = {r["s"]: r for r in json.load(gzip.open(files[i - 1], "rt"))["rows"]}
        if not (OW / "bars10y" / day[:4] / f"{day}.json.gz").exists():
            continue
        prev_rows, n = day_lists(day, prev_rows)
        print(day, n, flush=True)
    print("STARLISTS_DONE")


if __name__ == "__main__":
    main()
