"""ev1_features.py -- [2026-11-04] EV1 study, step 1: every EV1 buy of the 10-year Rule Lab run (job 20261104-193157-ea088a) measured AT THE
SIGNAL BAR (finished bars only, no look-ahead): price structure, EMAs/VWAP, volume, buy/sell flow, the opening, premarket, the market
(SPY/IWM) and the day's in-play environment (the day's top 30). Outcome = what came first after the buy: +2% (f2=1) or -2% (f2=-1).
Output reports/ev1/ev1_rows.json (one row per trade). Memory-light: one day file at a time, 14-day volume profiles only for needed symbols."""
import gzip, json, sys
from collections import defaultdict, deque
from pathlib import Path
import numpy as np
LAB = Path("/var/www/screener/trade/reports/lab"); OW = Path("/var/www/screener/trade/reports/open_window"); HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(LAB)); sys.path.insert(0, "/var/www/screener/trade/reports/playbook"); sys.path.insert(0, "/var/www/screener/trade")
import lab_engine as LE
import v2_bar_replay as R
JOB = sys.argv[1] if len(sys.argv) > 1 else "20261104-193157-ea088a"
T = [json.loads(l) for l in open(LAB / "jobs" / JOB / "trades.jsonl")]
for k, t in enumerate(T): t.setdefault("n", k)
BYDAY = defaultdict(list)
for t in T: BYDAY[t["day"]].append(t)
NEED = {t["sym"] for t in T}
MK = json.load(gzip.open(HERE / "market_1m.json.gz", "rt"))
def mkt(sym, day):
    return {r[0]: r for r in MK[sym].get(day, [])}
SPYD = {d: rows for d, rows in MK["SPY"].items()}
SDAYS = sorted(SPYD)
def spy_daily(day):
    """SPY prior-day return, 20-day trend (close vs 20-day mean), 20-day mean daily range %"""
    i = SDAYS.index(day) if day in SPYD else None
    if i is None or i < 21: return {}
    cl = [SPYD[d][-1][4] for d in SDAYS[i - 21:i]]; rg = [(max(r[2] for r in SPYD[d]) / min(r[3] for r in SPYD[d]) - 1) * 100 for d in SDAYS[i - 20:i]]
    return dict(spy_prev=(cl[-1] / cl[-2] - 1) * 100, spy_trend20=(cl[-1] / np.mean(cl[-20:]) - 1) * 100, spy_rng20=float(np.mean(rg)),
                spy_gap=(SPYD[day][0][1] / cl[-1] - 1) * 100)
def mkt_at(sym, day, m):
    b = mkt(sym, day)
    if not b or 0 not in b: return {}
    o = b[0][1]; pv = v = 0.0; c = None
    for k in range(0, m + 1):
        r = b.get(k)
        if r: pv += (r[2] + r[3] + r[4]) / 3 * r[5]; v += r[5]; c = r[4]
    c5 = (b.get(m - 5) or [0, 0, 0, 0, c])[4]
    p = sym.lower()
    return {f"{p}_open": (c / o - 1) * 100, f"{p}_vwap": (c / (pv / v) - 1) * 100 if v else 0, f"{p}_5m": (c / c5 - 1) * 100 if c5 else 0}

lib_days = {p.name[5:15] for p in OW.glob("bars/bars_20*.json.gz")}
b10 = LE.bars10y_days(OW)
PROF = {}                                                    # bars10y day -> {sym: 390 volume-by-minute (float32)} for the needed symbols
rows = []
days = sorted(BYDAY)
LIMIT = int(sys.argv[2]) if len(sys.argv) > 2 else None          # test: first N days (+ the last 2 = library days) -> ev1_rows_test.json
if LIMIT: days = days[:LIMIT] + days[-2:]
for day in days:
    if day < LE.LIB_START:                                  # 14-day normal volume by minute = the 14 bars10y days before this day
        prev = [x for x in b10 if x < day][-14:]
        for dd in list(PROF):
            if dd not in prev: del PROF[dd]
        for dd in prev:
            if dd not in PROF:
                BB = json.load(gzip.open(OW / "bars10y" / dd[:4] / f"{dd}.json.gz", "rt"))["bars"]
                PROF[dd] = {q: np.asarray(LE.vol_profile(BB[q], R), dtype=np.float32) for q in NEED if q in BB}; del BB
        HIST = defaultdict(list)
        for dd in prev:
            for q, v in PROF[dd].items(): HIST[q].append(v)
    # the day's stocks
    if day in lib_days:
        d = json.load(gzip.open(OW / f"bars/bars_{day}.json.gz", "rt")); uni = d["top30_current"]; RAW = {s: d["symbols"][s]["bars"] for s in uni if s in d["symbols"]}
        pp = OW / f"pm_history/{day}.json.gz"; prow = {r["s"]: r for r in json.load(gzip.open(pp, "rt"))["rows"]} if pp.exists() else {}
        REFS = {}
        for s in RAW:
            try: REFS[s] = R.REF.load(day, s)
            except Exception: REFS[s] = None
    else:
        prow = {r["s"]: r for r in json.load(gzip.open(OW / f"pm_history/{day}.json.gz", "rt"))["rows"]}
        BB = json.load(gzip.open(OW / "bars10y" / day[:4] / f"{day}.json.gz", "rt"))["bars"]
        uni = sorted([s for s in prow if s in BB and prow[s].get("prev_close")], key=lambda k: LE.v1(prow[k]), reverse=True)[:30]
        RAW = {s: BB[s] for s in uni}; del BB
        REFS = {s: {"prev_close": prow[s]["prev_close"], "atr14": prow[s].get("atr"),
                    "vol_per_min": [float(x) for x in np.mean(HIST[s], axis=0)] if len(HIST[s]) >= 5 else None} for s in uni}
    BARS = {s: [x for x in R.bars_of(raw) if 0 <= R.minute(x["t"]) < 390] for s, raw in RAW.items()}
    gaps = [(b[0]["o"] / REFS[s]["prev_close"] - 1) * 100 for s, b in BARS.items() if b and REFS.get(s) and REFS[s].get("prev_close")]
    sd = spy_daily(day)
    for t in BYDAY[day]:
        s = t["sym"]; b = BARS.get(s); ref = REFS.get(s)
        if not b or not ref: continue
        M = [R.minute(x["t"]) for x in b]
        if t["sig"] not in M: continue
        i = M.index(t["sig"]); m = t["sig"]
        C = [x["c"] for x in b]; H = [x["h"] for x in b]; L = [x["l"] for x in b]; V = [x["v"] for x in b]
        EF, ES = LE.ema(C, 9), LE.ema(C, 20)
        pv = cv = 0.0
        for x in b[:i + 1]: pv += (x["h"] + x["l"] + x["c"]) / 3 * x["v"]; cv += x["v"]
        vw = pv / cv; c = C[i]; o = b[0]["o"]; pc = ref["prev_close"]; atr_d = ref["atr14"] or o * 0.05
        hi, lo = max(H[:i + 1]), min(L[:i + 1]); lo_i = L.index(lo)
        fp = (OW / "flow" / day / f"{s}.json") if day in lib_days else (OW / "flow_tick" / day / f"{s}.json")
        fl = json.loads(fp.read_text()) if fp.exists() else None
        def share(k):
            if not fl: return None
            a_ = sum(fl["buy"][max(0, m - k + 1):m + 1]); z_ = sum(fl["sell"][max(0, m - k + 1):m + 1]); return a_ / (a_ + z_) if a_ + z_ else None
        vpm = ref.get("vol_per_min")
        rv = sum(V[:i + 1]) / max(sum(vpm[:m + 1]), 1) if vpm else None
        up = 0
        for k in range(i, 0, -1):
            if C[k] > C[k - 1]: up += 1
            else: break
        o5 = [x for x in b if R.minute(x["t"]) < 5]
        rng_b = H[i] - L[i]
        pmh = (prow.get(s) or {}).get("pm_high")
        top_up = np.mean([BARS[q][min(len(BARS[q]) - 1, next((k for k, x in enumerate(BARS[q]) if R.minute(x["t"]) >= m), len(BARS[q]) - 1))]["c"] > BARS[q][0]["o"]
                          for q in BARS if BARS[q]])
        r = dict(n=t["n"], day=day, year=int(day[:4]), sym=s, f2=t["f2"], r60=t["r60"], d30=t["d30"], pl=t["pl"],
                 t_min=m, gap=t["gap"], atr=t["atr"], price=c,
                 ext_open=(c / o - 1) * 100, c_vs_vwap=(c / vw - 1) * 100, ema_vs_vwap=(EF[i] / vw - 1) * 100, ema9_20=(EF[i] / ES[i] - 1) * 100,
                 ema9_slope3=(EF[i] / EF[max(0, i - 3)] - 1) * 100, vwap_3m=t["read"].get("vwap_3m"), vwap_30m=t["read"].get("vwap_30m"),
                 range_atr=(hi - lo) / atr_d, run_from_low_atr=(c - lo) / atr_d, below_high=(c / hi - 1) * 100, low_min=M[lo_i],
                 daypos=(c - lo) / max(hi - lo, 1e-9), open5_atr=((max(x["h"] for x in o5) - min(x["l"] for x in o5)) / atr_d) if o5 else None,
                 open5_dir=((o5[-1]["c"] / o - 1) * 100) if o5 else None,
                 ret5=(c / C[max(0, i - 5)] - 1) * 100, ret15=(c / C[max(0, i - 15)] - 1) * 100, up_bars=up,
                 bar_atr=rng_b / atr_d, bar_close_pos=(c - L[i]) / rng_b if rng_b else 0.5, upper_wick=(H[i] - max(c, b[i]["o"])) / rng_b if rng_b else 0,
                 vol_x=t["read"].get("vol_x"), rvol=rv, vol_trend=(sum(V[i - 4:i + 1]) / 5) / max(sum(V[max(0, i - 14):i - 4]) / 10, 1) if i >= 14 else None,
                 s3=share(3), s15=share(15), s_open=share(m + 1), room=t["read"].get("room"), dip_atr=t["read"].get("dip_atr"),
                 chg_60m=t["read"].get("chg_60m"), long_ema_10m=t["read"].get("long_ema_10m"), liq=t["read"].get("liq"),
                 vs_pmh=((c / pmh - 1) * 100) if pmh else None, prev_close_gap=(c / pc - 1) * 100,
                 top30_up=float(top_up), top30_gap_med=float(np.median(gaps)) if gaps else None, signals_day=len(BYDAY[day]))
        # [11/04 user] the LAST 15 MINUTES before the signal (the path into it, not only the values at the signal bar)
        VWA = []; pv2 = cv2 = 0.0
        for x in b[:i + 1]: pv2 += (x["h"] + x["l"] + x["c"]) / 3 * x["v"]; cv2 += x["v"]; VWA.append(pv2 / cv2 if cv2 else x["c"])
        w0 = max(0, i - 14); W = range(w0, i + 1); nW = i + 1 - w0
        cw = np.array([C[k] for k in W]); path_ = sum(abs(C[k] - C[k - 1]) for k in W if k > 0)
        tt = np.arange(nW); r2 = float(np.corrcoef(tt, cw)[0, 1] ** 2) if nW >= 3 and cw.std() > 0 else 0.0
        run_hi, pb = 0.0, 0.0
        for k in W:
            run_hi = max(run_hi, H[k]); pb = min(pb, (L[k] / run_hi - 1) * 100)
        hw = max(H[k] for k in W); lw = min(L[k] for k in W)
        vols = [V[k] for k in W]
        def sh15(mm):
            if not fl or mm < 0: return None
            a_ = sum(fl["buy"][max(0, mm - 14):mm + 1]); z_ = sum(fl["sell"][max(0, mm - 14):mm + 1]); return a_ / (a_ + z_) if a_ + z_ else None
        s15_then = sh15(m - 15); s15_now = share(15)
        s3s = [x for x in (((sum(fl["buy"][max(0, M[k] - 2):M[k] + 1]) / max(sum(fl["buy"][max(0, M[k] - 2):M[k] + 1]) + sum(fl["sell"][max(0, M[k] - 2):M[k] + 1]), 1))
                            if fl else None) for k in W) if x is not None]
        sellmax = max((fl["sell"][M[k]] for k in W), default=0) if fl else None
        r.update(w_range_atr=(hw - lw) / atr_d, w_straight=abs(C[i] - C[w0]) / path_ if path_ else 0, w_r2=r2,
                 w_green=sum(C[k] > b[k]["o"] for k in W) / nW, w_pullback=pb, w_since_low=i - max(W, key=lambda k: -L[k]),
                 w_since_high=i - max(W, key=lambda k: H[k]), w_below_vwap=sum(C[k] < VWA[k] for k in W), w_vwap_cross=int(any(C[k] <= VWA[k] for k in W)),
                 w_ema_cross=sum((C[k] - EF[k]) * (C[k - 1] - EF[k - 1]) < 0 for k in W if k > 0),
                 w_vwap_slope=(VWA[i] / VWA[w0] - 1) * 100, w_ema_slope=(EF[i] / EF[w0] - 1) * 100,
                 w_maxbar_atr=max(H[k] - L[k] for k in W) / atr_d, w_vol_spike=max(vols) / max(np.median(vols), 1),
                 w_share_trend=(s15_now - s15_then) if (s15_now is not None and s15_then is not None) else None,
                 w_share_min=min(s3s) if s3s else None, w_sell_flush=(sellmax / max(np.mean(vols), 1)) if fl else None,
                 w_new_high=int(hw >= max(H[:w0 + 1])), w_near_vwap=min((C[k] / VWA[k] - 1) * 100 for k in W))
        r.update(sd); r.update(mkt_at("SPY", day, m)); r.update(mkt_at("IWM", day, m))
        rows.append(r)
    del RAW, BARS; R.REF._CACHE.clear()
    print(day, len(rows), flush=True)
json.dump(rows, open(HERE / ("ev1_rows_test.json" if LIMIT else "ev1_rows.json"), "w")); print("FEATURES_DONE", len(rows))
