"""
premarket_history.py -- [2026-10-06] PRIORITY (the user): 2 years of the scanner's 9:28 inputs for EVERY universe
stock, to re-weight the scanner score on a large sample (the 20-day study: daily ATR% and gap size predict movers;
signed gap / premarket strength carry no information).
Per trading day D (Massive, Developer plan):
  universe  ~ the live prefilter: previous close $4-16, previous-day volume >= 500k, plain tickers; the last
            premarket price must be $5-15 (live: last premarket trade $5-15)
  inputs    premarket 1-min bars 4:00-9:28: volume, last price, high, low, early/late strength (scanner formula),
            gap vs previous close; previous-day high, 20-day high, daily ATR(14) and 20-day average volume from
            Massive grouped daily bars; distance to the nearest resistance (scanner formula)
  outcome   that day's open / high / low / close / volume (grouped daily)
  output    reports/open_window/pm_history/<D>.json.gz  {"date", "rows": [{...}]}
Resumable; 8 requests in parallel; never runs 9:10-16:05 on weekdays.
    python3 premarket_history.py [START] [END]       default 2024-10-01 .. yesterday
"""
import gzip
import json
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import date, datetime, timedelta
from pathlib import Path
from zoneinfo import ZoneInfo

sys.path.insert(0, str(Path(__file__).resolve().parent))
from massive_trial import get, BASE

ET = ZoneInfo("America/New_York")
HERE = Path(__file__).resolve().parent
OUT = HERE / "pm_history"
GCACHE = HERE / "pm_history" / "grouped"
COMMON = set(json.load(open(HERE / "common_stocks.json")))     # Massive type CS, active + delisted


def market_hours():
    t = datetime.now(ET)
    return t.weekday() < 5 and (9, 10) <= (t.hour, t.minute) < (16, 5)


def grouped(d):
    f = GCACHE / f"{d}.json.gz"
    if f.exists():
        return json.load(gzip.open(f, "rt"))
    j = get(f"{BASE}/v2/aggs/grouped/locale/us/market/stocks/{d}", {"adjusted": "false"})
    out = {r["T"]: [r["o"], r["h"], r["l"], r["c"], r["v"]] for r in j.get("results", [])
           if all(k in r for k in "ohlcv") and 1 <= r["c"] <= 40}
    GCACHE.mkdir(parents=True, exist_ok=True)
    with gzip.open(f, "wt") as fh:
        json.dump(out, fh)
    return out


def premarket(sym, d):
    j = get(f"{BASE}/v2/aggs/ticker/{sym}/range/1/minute/{d}/{d}", {"adjusted": "false", "sort": "asc", "limit": 50000})
    d0 = datetime.fromisoformat(str(d)).replace(tzinfo=ET)
    a, z = d0.replace(hour=4).timestamp() * 1000, d0.replace(hour=9, minute=28).timestamp() * 1000
    return [b for b in j.get("results", []) if a <= b["t"] < z]


def strength(bars):                       # scanner._premarket_price_strength
    n = len(bars)
    if n < 3:
        return 0.0
    third = max(1, n // 3)
    e = sum(b["c"] for b in bars[:third]) / third
    l = sum(b["c"] for b in bars[-third:]) / third
    return (l / e - 1) * 100 if e else 0.0


def main(start="2024-10-01", end=None):
    end = end or str(date.today() - timedelta(days=1))
    days, d = [], date.fromisoformat(start) - timedelta(days=45)
    while d <= date.fromisoformat(end):
        if d.weekday() < 5:
            days.append(d)
        d += timedelta(days=1)
    hist = []                                  # (date, grouped) of past trading days
    t0, n = time.time(), 0
    for d in days:
        if market_hours():
            print("stopped: market hours", flush=True); return
        g = grouped(d)
        if not g:
            continue                           # holiday
        if str(d) >= start and len(hist) >= 20:
            f = OUT / f"{d}.json.gz"
            if not f.exists():
                prev = hist[-1][1]
                past = [h[1] for h in hist[-20:]]
                cands = [s for s, v in prev.items() if 4 <= v[3] <= 16 and v[4] >= 500_000 and s.isalpha() and len(s) <= 5
                         and s in COMMON]           # common stocks only (no ETFs / funds), like the live universe

                def one(s):
                    try:
                        pm = premarket(s, d)
                    except Exception:
                        return None
                    if len(pm) < 3:
                        return None
                    last = pm[-1]["c"]
                    if not 5 <= last <= 15 or s not in g:
                        return None
                    pc, ph = prev[s][3], prev[s][1]
                    hs = [p[s] for p in past if s in p]
                    trs, pcl = [], None
                    for b in hs[-15:]:
                        if pcl is not None:
                            trs.append(max(b[1] - b[2], abs(b[1] - pcl), abs(b[2] - pcl)))
                        pcl = b[3]
                    h20 = max(b[1] for b in hs) if hs else None
                    pmh = max(b["h"] for b in pm)
                    dist = None
                    for lvl in (pmh, ph, h20):
                        if lvl and lvl > last:
                            dist = (lvl - last) / lvl * 100; break
                    o, h, l, c, v = g[s]
                    return {"s": s, "pm_vol": sum(b["v"] for b in pm), "pm_last": last, "pm_high": pmh,
                            "pm_low": min(b["l"] for b in pm), "pm_bars": len(pm), "strength": round(strength(pm), 3),
                            "gap": round((last / pc - 1) * 100, 3), "prev_close": pc, "prev_high": ph, "high20": h20,
                            "atr": round(sum(trs) / len(trs), 4) if trs else None,
                            "adv20": round(sum(b[4] for b in hs) / len(hs)) if hs else None,
                            "dist": round(dist, 3) if dist is not None else None,
                            "o": o, "h": h, "l": l, "c": c, "v": v}

                with ThreadPoolExecutor(8) as ex:
                    rows = [r for r in ex.map(one, cands) if r]
                tmp = f.with_suffix(".tmp")
                with gzip.open(tmp, "wt") as fh:
                    json.dump({"date": str(d), "rows": rows}, fh)
                tmp.rename(f)
                n += 1
                print(f"{d}: {len(cands)} prefiltered -> {len(rows)} in the $5-15 premarket universe "
                      f"[{(time.time() - t0) / 60:.0f} min, {n} days]", flush=True)
        hist.append((d, g))
        hist = hist[-25:]
    print("PM_HISTORY_DONE", flush=True)


if __name__ == "__main__":
    main(*sys.argv[1:3])
