"""book_study.py -- [2026-10-27] the user: does the buyer/seller QUEUE (bid size vs ask size, data/book recorder, 10/12-10/26) predict
the next minutes' flow and price?  Stage 1 (cache): per day, every book stock: Alpaca 1-min bars + Massive tick-rule flow -> book_cache/<day>.json
Stage 2: per finished minute t: queue readings (imb, imb5 = 5-min avg, bid/ask moves) vs NEXT 5 / 15 min buy share and price
(buy at t+1 open). First half of the days vs second half. No look-ahead: readings use minutes <= t only."""
import glob, json, sys
from datetime import datetime
from pathlib import Path
from zoneinfo import ZoneInfo
import numpy as np
HERE = Path(__file__).resolve().parent; TR = Path("/var/www/screener/trade"); C = HERE / "book_cache"
for p in (str(TR), str(TR / "reports/open_window")):
    sys.path.insert(0, p)
ET = ZoneInfo("America/New_York")
DAYS = sorted(Path(f).stem for f in glob.glob(str(TR / "data/book/*.jsonl")))
def book(day):
    B = {}
    for line in open(TR / f"data/book/{day}.jsonl"):
        r = json.loads(line); hh, mm = map(int, r["minute"].split(":")); m = (hh - 9) * 60 + mm - 30
        if 0 <= m < 390: B.setdefault(r["sym"], {})[m] = r
    return B
if sys.argv[1] == "cache":
    from alpaca.data.requests import StockBarsRequest
    from alpaca.data.timeframe import TimeFrame
    from alpaca.data.historical import StockHistoricalDataClient
    from config_loader import get_env
    from massive_trial import m_trades, tick_flow
    env = get_env(); hc = StockHistoricalDataClient(env.api_key, env.secret_key)
    for day in DAYS:
        out = C / f"{day}.json"
        if out.exists(): continue
        syms = sorted(book(day)); d0 = datetime.fromisoformat(day)
        raw = hc.get_stock_bars(StockBarsRequest(symbol_or_symbols=syms, timeframe=TimeFrame.Minute, start=d0.replace(hour=9, minute=30, tzinfo=ET),
                                                 end=d0.replace(hour=16, tzinfo=ET))).data
        R = {}
        for s in syms:
            if s not in raw: continue
            bars = {}
            for x in raw[s]:
                t = x.timestamp.astimezone(ET); m = (t.hour - 9) * 60 + t.minute - 30
                if 0 <= m < 390: bars[m] = [x.open, x.high, x.low, x.close, x.volume]
            try:
                fb, fs = tick_flow(m_trades(s, day))
            except Exception as e:
                print("skip", day, s, e, flush=True); continue
            R[s] = {"bars": bars, "buy": [float(v) for v in fb], "sell": [float(v) for v in fs]}
        json.dump(R, open(out, "w")); print(day, len(R), flush=True)
    print("CACHE_DONE"); sys.exit()
# ---------------- stage 2 ----------------
rows = []
for day in DAYS:
    Bk = book(day); R = json.load(open(C / f"{day}.json"))
    for s, r in R.items():
        bk = Bk.get(s, {}); bars = {int(k): v for k, v in r["bars"].items()}; fb, fs = np.array(r["buy"]), np.array(r["sell"])
        hi = -1e9
        for t in range(390):
            if t in bars: hi = max(hi, bars[t][1])
            if t < 5 or t not in bars or t > 370 or t not in bk or t + 1 not in bars: continue
            last5 = [bk[k] for k in range(t - 4, t + 1) if k in bk]
            if len(last5) < 4: continue
            P = bars[t + 1][0]
            f5 = [bars[k][3] for k in range(t + 1, t + 6) if k in bars]; f15 = [bars[k][3] for k in range(t + 1, t + 16) if k in bars]
            if not f5 or len(f15) < 10: continue
            def sh(a, b):
                x, y = fb[a:b].sum(), fs[a:b].sum(); return x / (x + y) if x + y > 0 else np.nan
            rows.append(dict(day=day, sym=s, t=t, imb=bk[t]["imbalance"], imb5=np.mean([z["imbalance"] for z in last5]),
                             bmove=sum(z["bid_up"] - z["bid_dn"] for z in last5), amove=sum(z["ask_up"] - z["ask_dn"] for z in last5),
                             spread=bk[t]["spread_pct"], bs5=sh(t - 4, t + 1), bs15=sh(max(0, t - 14), t + 1),
                             off_high=(bars[t][3] / hi - 1) * 100, fbs5=sh(t + 1, t + 6), fbs15=sh(t + 1, t + 16),
                             r5=(f5[-1] / P - 1) * 100, r15=(f15[-1] / P - 1) * 100, dn15=(min(bars[k][2] for k in range(t + 1, t + 16) if k in bars) / P - 1) * 100))
half = DAYS[len(DAYS) // 2]
H1 = [x for x in rows if x["day"] < half]; H2 = [x for x in rows if x["day"] >= half]
print(f"{len(rows)} stock-minutes, {len(DAYS)} days | half 1 = {DAYS[0]}..< {half} ({len(H1)}), half 2 = {half}.. ({len(H2)})")
def spear(X, a, b):
    v = np.array([(x[a], x[b]) for x in X if np.isfinite(x[a]) and np.isfinite(x[b])])
    ra = v[:, 0].argsort().argsort(); rb = v[:, 1].argsort().argsort(); return np.corrcoef(ra, rb)[0, 1]
print("\n(1) rank correlation of the reading NOW with the NEXT minutes (half 1 / half 2); 0 = no link, +-0.05 = weak, +-0.1 = useful")
for a in ("imb", "imb5", "bmove", "amove", "bs5", "bs15"):
    print(f"   {a:6} -> next 5m buy share {spear(H1, a, 'fbs5'):+.3f}/{spear(H2, a, 'fbs5'):+.3f} | next 15m buy share {spear(H1, a, 'fbs15'):+.3f}/{spear(H2, a, 'fbs15'):+.3f}"
          f" | next 5m price {spear(H1, a, 'r5'):+.3f}/{spear(H2, a, 'r5'):+.3f} | next 15m price {spear(H1, a, 'r15'):+.3f}/{spear(H2, a, 'r15'):+.3f}")
def tab(X):
    return f"n {len(X):6} | next15 buy {np.nanmean([x['fbs15'] for x in X]) * 100:5.1f}% | price 5m {np.mean([x['r5'] for x in X]):+.3f}% 15m {np.mean([x['r15'] for x in X]):+.3f}% | up>1% {np.mean([x['r15'] > 1 for x in X]):4.0%} down<-1% {np.mean([x['dn15'] < -1 for x in X]):4.0%}" if X else "-"
print("\n(2) queue 5-min average in 5 groups (cuts from half 1):")
q = np.quantile([x["imb5"] for x in H1], [0.2, 0.4, 0.6, 0.8]); print("   cuts", np.round(q, 3))
for gi in range(5):
    g = lambda x: int(np.searchsorted(q, x["imb5"])) == gi
    print(f"   group {gi + 1}  H1 {tab([x for x in H1 if g(x)])}\n            H2 {tab([x for x in H2 if g(x)])}")
print("\n(3) does the queue add anything beyond the traded buy share? same 15-min buy share, queue split:")
for lo, hi_, lab in ((0, 0.45, "buy share 15m < 45%"), (0.45, 0.55, "45-55%"), (0.55, 1.01, ">= 55%")):
    for qa, qb, ql in ((-1.01, -0.2, "queue <= -0.2 (sellers waiting)"), (-0.2, 0.2, "queue -0.2..+0.2"), (0.2, 1.01, "queue >= +0.2 (buyers waiting)")):
        f = lambda x: lo <= x["bs15"] < hi_ and qa < x["imb5"] <= qb
        print(f"   {lab:20} {ql:32} H1 {tab([x for x in H1 if f(x)])}\n   {'':53} H2 {tab([x for x in H2 if f(x)])}")
print("\n(4) the two WULF cases:")
A = lambda x: x["imb5"] >= 0.3 and x["bs15"] < 0.45; A0 = lambda x: x["imb5"] < 0.3 and x["bs15"] < 0.45
print(f"   A mismatch: queue >= +0.3 but buy share 15m < 45%   H1 {tab([x for x in H1 if A(x)])}\n   {'':52} H2 {tab([x for x in H2 if A(x)])}")
print(f"   A compare: buy share < 45%, queue < +0.3           H1 {tab([x for x in H1 if A0(x)])}\n   {'':52} H2 {tab([x for x in H2 if A0(x)])}")
N = lambda x: x["off_high"] > -0.5 and x["imb5"] <= -0.2; N0 = lambda x: x["off_high"] > -0.5 and x["imb5"] > -0.2
print(f"   B near the high (<0.5%) + queue <= -0.2              H1 {tab([x for x in H1 if N(x)])}\n   {'':52} H2 {tab([x for x in H2 if N(x)])}")
print(f"   B compare: near the high, queue > -0.2              H1 {tab([x for x in H1 if N0(x)])}\n   {'':52} H2 {tab([x for x in H2 if N0(x)])}")
print("BOOK_STUDY_DONE")
