# [2026-10-04] how similar are the 30 closest good days to the qualified stocks? (after the 0-trade replay at 90%)
import sys, json, gzip, numpy as np
sys.path.insert(0, "/var/www/screener/premarket")
import star_live, star_replay as R
from datetime import datetime, timedelta
eng = star_live.StarEngine({}, today="2099-01-01")
days = sorted(p.name[15:25] for p in (R.OW / "tick20/out").glob("entry1002_live_*.out"))
best = []; atq = []; allm = []
for day in days:
    d = json.load(gzip.open(R.OW / "bars" / f"bars_{day}.json.gz", "rt"))
    bj = json.load(gzip.open(R.OW / "bars" / f"bench_{day}.json.gz", "rt"))
    bench = (R.rows_to_bars(bj["SPY"]), R.rows_to_bars(bj["IWM"]))
    q = R.qualified(day); eng.set_day(day)
    for s, mq in q.items():
        rp = R.REFD / day / f"{s}.json"; fp = R.OW / "flow" / day / f"{s}.json"
        if s not in d["symbols"] or not rp.exists() or not fp.exists():
            continue
        fl = json.loads(fp.read_text()); bars = R.rows_to_bars(d["symbols"][s]["bars"])
        flow = {m: (fl["buy"][m], fl["sell"][m]) for m in range(390)}
        now = datetime.fromisoformat(day).replace(tzinfo=R.ET).replace(hour=9, minute=30) + timedelta(minutes=345)
        eng.update(s, bars, json.loads(rp.read_text()), bench, flow, now)   # all minutes to 15:14 at once
        # replay minute by minute the 30th-closest similarity: recompute cumulative D per minute
        F, C, L = eng.features(bars, json.loads(rp.read_text()), bench, flow, 344)
        D = np.zeros(eng.Fm.shape[1]); sims = []
        for m in range(345):
            D += ((eng.Fm[m] - F[m][None, :]) ** 2).sum(1)
            if m >= mq:
                mean = D / (m + 1) + eng.block
                d30 = np.partition(mean, 29)[29]
                sims.append(1 - d30 / eng.typical[m])
                allm.append(sims[-1])
        if sims:
            best.append(max(sims)); atq.append(sims[0])
best, atq, allm = np.array(best), np.array(atq), np.array(allm)
print(f"{len(best)} qualified stocks")
print("similarity of the 30th-closest good day AT qualification: median %.2f, p90 %.2f, max %.2f" % (np.median(atq), np.percentile(atq, 90), atq.max()))
print("best similarity reached later that day:                   median %.2f, p90 %.2f, max %.2f" % (np.median(best), np.percentile(best, 90), best.max()))
for t in (0.9, 0.85, 0.8, 0.75, 0.7, 0.6, 0.5):
    print(f"  30 refs at >= {t:.0%} similar: stocks {(best >= t).sum()}/{len(best)}, stock-minutes {(allm >= t).mean():.1%}")
