"""
massive_bars_10y.py -- [2026-10-05] 10 YEARS of 1-minute bars (Massive, ex-Polygon, Developer plan) for every
US stock that opened at $5-15 with >= 500k shares that day (~500/day), to choose playbook references from
later (the user: download 10 years, select later). Trades/flow are NOT downloaded here (too big for all):
only for the selected stock-days, later.
  per day: grouped daily bars (1 call) -> list; then 1 minute-aggregate call per stock (8 in parallel)
  output: reports/open_window/bars10y/<YYYY>/<date>.json.gz  {"date", "daily": {sym: [o,h,l,c,v,prev_close]},
          "bars": {sym: [[epoch_s, o, h, l, c, v], ...] 9:30-16:00}}
Resumable (skips finished days), newest day first. Never runs 9:10-16:05 on weekdays (market hours).
    python3 massive_bars_10y.py [START] [END]      default 2016-10-03 .. 2026-10-02
"""
import gzip
import json
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import date, datetime, timedelta
from pathlib import Path
from zoneinfo import ZoneInfo

sys.path.insert(0, str(Path(__file__).resolve().parent))
from massive_trial import get, BASE

ET = ZoneInfo("America/New_York")
OUT = Path(__file__).resolve().parent / "bars10y"


def market_hours():
    t = datetime.now(ET)
    return t.weekday() < 5 and (9, 10) <= (t.hour, t.minute) < (16, 5)


def grouped(d):
    j = get(f"{BASE}/v2/aggs/grouped/locale/us/market/stocks/{d}", {"adjusted": "false"})
    return {r["T"]: r for r in j.get("results", []) if all(k in r for k in ("o", "h", "l", "c", "v"))}


def minute_bars(sym, d):
    j = get(f"{BASE}/v2/aggs/ticker/{sym}/range/1/minute/{d}/{d}", {"adjusted": "false", "sort": "asc", "limit": 50000})
    d0 = datetime.fromisoformat(str(d)).replace(tzinfo=ET)
    a, z = d0.replace(hour=9, minute=30).timestamp(), d0.replace(hour=16).timestamp()
    return [[int(b["t"] / 1000), b["o"], b["h"], b["l"], b["c"], b["v"]] for b in j.get("results", [])
            if a <= b["t"] / 1000 < z]


def main(start="2016-10-03", end="2026-10-02"):
    days, d = [], date.fromisoformat(end)
    while d >= date.fromisoformat(start):
        if d.weekday() < 5:
            days.append(d)
        d -= timedelta(days=1)
    prev = None
    t0, n = time.time(), 0
    for d in days:
        f = OUT / str(d.year) / f"{d}.json.gz"
        if f.exists():
            continue
        if market_hours():
            print("stopped: market hours", flush=True); return
        try:
            today = grouped(d)
        except RuntimeError as e:
            if "NOT_AUTHORIZED" in str(e):
                # [2026-10-15] the plan serves a rolling 10 years: older days are not available -> download complete
                print(f"{d}: older than the plan's 10-year history limit -- nothing more to download", flush=True)
                print("BARS10Y_DONE", flush=True); return
            raise
        if not today:
            continue                                   # holiday
        # previous trading day's closes (for gaps)
        pd_, pc = d - timedelta(days=1), {}
        for _ in range(6):
            if pd_.weekday() < 5:
                g = grouped(pd_)
                if g:
                    pc = {k: v["c"] for k, v in g.items()}
                    break
            pd_ -= timedelta(days=1)
        syms = [s for s, r in today.items() if 5 <= r["o"] <= 15 and r["v"] >= 500_000 and s.isalpha()]
        with ThreadPoolExecutor(8) as ex:
            bars = dict(zip(syms, ex.map(lambda s: minute_bars(s, d), syms)))
        f.parent.mkdir(parents=True, exist_ok=True)
        tmp = f.with_suffix(".tmp")
        with gzip.open(tmp, "wt") as fh:
            json.dump({"date": str(d),
                       "daily": {s: [today[s]["o"], today[s]["h"], today[s]["l"], today[s]["c"], today[s]["v"], pc.get(s)] for s in syms},
                       "bars": {s: b for s, b in bars.items() if b}}, fh)
        tmp.rename(f)
        n += 1
        print(f"{d}: {len(syms)} stocks [{(time.time() - t0) / 60:.0f} min, {n} days]", flush=True)
    print("BARS10Y_DONE", flush=True)


if __name__ == "__main__":
    main(*sys.argv[1:3])
