"""
massive_trial.py -- [2026-10-05] Trial of Massive (ex-Polygon) Developer data vs our Alpaca data, before the
5-year download (the user subscribed; key MASSIVE_API_KEY in trade/.env, never printed).
For each stock of a day's list (bars/bars_<day>.json.gz):
  bars  Massive 1-min aggregates vs our Alpaca bars: volume and close agreement
  flow  Massive trades -> buy/sell per minute by the TICK RULE vs our flow file (quote rule + tick
        fallback): correlation of the 15-min net-flow channel (lib_v3 FL) and of per-minute net flow
    python3 massive_trial.py 2026-09-18 2026-10-01
"""
import gzip
import json
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from zoneinfo import ZoneInfo

import numpy as np
import requests

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parents[1] / "reports" / "playbook"))
import lib_v3

ET = ZoneInfo("America/New_York")
BASE = "https://api.massive.com"


def key():
    for l in open("/var/www/screener/trade/.env"):
        if l.upper().startswith("MASSIVE_API_KEY="):
            return l.split("=", 1)[1].strip().strip("'\"")


S = requests.Session()
S.headers["Authorization"] = f"Bearer {key()}"


def get(url, params=None):
    for k in range(5):
        r = S.get(url, params=params, timeout=60)
        if r.status_code == 200:
            return r.json()
        time.sleep(2 + 3 * k)
    raise RuntimeError(f"{r.status_code} {r.text[:200]}")


def minute(ts_ns):
    t = datetime.fromtimestamp(ts_ns / 1e9, timezone.utc).astimezone(ET)
    return (t.hour - 9) * 60 + t.minute - 30


def m_bars(sym, day):
    j = get(f"{BASE}/v2/aggs/ticker/{sym}/range/1/minute/{day}/{day}", {"adjusted": "false", "sort": "asc", "limit": 50000})
    out = {}
    for b in j.get("results", []):
        m = minute(b["t"] * 1_000_000)
        if 0 <= m < 390:
            out[m] = (b["o"], b["h"], b["l"], b["c"], b["v"])
    return out


def m_trades(sym, day):
    d0 = datetime.fromisoformat(day).replace(tzinfo=ET)
    a = int(d0.replace(hour=9, minute=30).timestamp() * 1e9); z = int(d0.replace(hour=16).timestamp() * 1e9)
    url, params, out = f"{BASE}/v3/trades/{sym}", {"timestamp.gte": a, "timestamp.lt": z, "limit": 50000, "sort": "timestamp", "order": "asc"}, []
    while url:
        j = get(url, params)
        out += [(t["sip_timestamp"], t["price"], t["size"]) for t in j.get("results", [])]
        url, params = j.get("next_url"), None
    return out


def tick_flow(trades):
    buy, sell = np.zeros(390), np.zeros(390)
    last_p, last_side = None, 0
    for ts, p, s in trades:
        side = 1 if last_p is not None and p > last_p else -1 if last_p is not None and p < last_p else last_side
        last_p, last_side = p, side
        m = minute(ts)
        if 0 <= m < 390:
            if side > 0:
                buy[m] += s
            elif side < 0:
                sell[m] += s
    return buy, sell


def main(days):
    rows = []
    for day in days:
        d = json.load(gzip.open(HERE / "bars" / f"bars_{day}.json.gz", "rt"))
        t0 = time.time()
        for s in d["top30_current"]:
            fp = HERE / "flow" / day / f"{s}.json"
            if s not in d["symbols"] or not fp.exists():
                continue
            ours = {}
            for r in d["symbols"][s]["bars"]:
                t = r[0] if isinstance(r[0], (int, float)) else r[0]
                m = minute(t * 1e9) if isinstance(t, (int, float)) else None
                if m is not None and 0 <= m < 390:
                    ours[m] = r[1:6]
            mb = m_bars(s, day)
            common = sorted(set(ours) & set(mb))
            if len(common) < 100:
                continue
            vo = np.array([ours[m][4] for m in common]); vm = np.array([mb[m][4] for m in common])
            co = np.array([ours[m][3] for m in common]); cm = np.array([mb[m][3] for m in common])
            fl = json.loads(fp.read_text())
            tr = m_trades(s, day)
            mbuy, msell = tick_flow(tr)
            fo = lib_v3.flow_channel(fl["buy"], fl["sell"]); fm = lib_v3.flow_channel(mbuy, msell)
            no = np.array(fl["buy"]) - np.array(fl["sell"]); nm = mbuy - msell
            ok = (np.abs(no) + np.abs(nm)) > 0
            rows.append({"day": day, "sym": s, "minutes": len(common),
                         "vol_ratio": float(vm.sum() / max(vo.sum(), 1)),
                         "vol_corr": float(np.corrcoef(vo, vm)[0, 1]),
                         "close_diff_pct": float(np.median(np.abs(cm / co - 1)) * 100),
                         "trades": len(tr),
                         "FL_corr": float(np.corrcoef(fo, fm)[0, 1]),
                         "FL_mad": float(np.mean(np.abs(fo - fm))),
                         "net_corr": float(np.corrcoef(no[ok], nm[ok])[0, 1]) if ok.sum() > 10 else None})
            print(f"{day} {s:5} vol x{rows[-1]['vol_ratio']:.3f} (corr {rows[-1]['vol_corr']:.3f}) close diff {rows[-1]['close_diff_pct']:.3f}% "
                  f"| {len(tr)} trades, flow channel corr {rows[-1]['FL_corr']:.2f}, mean diff {rows[-1]['FL_mad']:.3f}", flush=True)
        print(f"{day}: {time.time() - t0:.0f} s for {len(d['top30_current'])} stocks", flush=True)
    R = rows
    for k in ("vol_ratio", "vol_corr", "close_diff_pct", "FL_corr", "FL_mad", "net_corr"):
        v = np.array([r[k] for r in R if r[k] is not None])
        print(f"{k:14}: median {np.median(v):.3f}  p10 {np.percentile(v, 10):.3f}  p90 {np.percentile(v, 90):.3f}")
    json.dump(R, open(HERE / "massive_trial.json", "w"), indent=1)


if __name__ == "__main__":
    main(sys.argv[1:])
