"""
build_history.py -- [2026-09-29] 6 months of test data (the user: "create the 6 month data,
so we can test it and add more patterns to the book"). For every trading day in the range,
rebuild the scanner's 9:28 top-30 the way the live scanner picks it, and save the same files
the other days have, so every study / simulator picks the days up automatically:

  bars/bars_<date>.json.gz     full-day 1-min bars (9:30-16:00) of the top 30 + levels
  bars/bench_<date>.json.gz    SPY / IWM 1-min bars
  tick20/cands/<date>.json     the list for the tick simulator (ticks download on first use)
  data/reference/<date>/       14-day reference files of the top 30 (reference.build)

List rebuild, per day (as the live bot: universe.prefilter_by_snapshot + scanner.scan):
  universe  today's tradable asset list (universe.get_universe_symbols -- CAVEAT: symbols
            delisted since are missing, so the lists lean slightly toward survivors)
  prefilter previous session volume >= min_avg_daily_volume; last premarket price
            (bars 3:30-9:28) in [price_min, price_max]
  score     scanner.score_breakout_candidate (the live function), >= 3 premarket bars
  top 30    by candidate_score
Not included: intraday rescans (the live bot adds stocks at 9:40, 10:10, ...), buy/sell
flow (needs tick data: tick20 simulator downloads it per day when it runs).

    python3 build_history.py 2026-03-02 2026-08-26      newest day first; resumable;
                                                        stops at 9:15 ET, never runs 9:15-16:05
"""
import gzip
import json
import sys
import time
from datetime import date, datetime, timedelta
from pathlib import Path
from zoneinfo import ZoneInfo

HERE = Path(__file__).resolve().parent
TRADE = HERE.parents[1]
sys.path.insert(0, str(TRADE))
from alpaca.data.requests import StockBarsRequest
from alpaca.data.timeframe import TimeFrame, TimeFrameUnit
from alpaca_client import get_client
from config_loader import get_config
import reference
import scanner
import universe
import volatility

ET = ZoneInfo("America/New_York")
BARS = HERE / "bars"
CANDS = HERE / "tick20" / "cands"
CACHE = HERE / "history"


def market_hours_now():
    t = datetime.now(ET)
    return t.weekday() < 5 and (9, 15) <= (t.hour, t.minute) < (16, 5)


def minute_bars(client, syms, start, end, chunk=50):
    out = {}
    for i in range(0, len(syms), chunk):
        req = StockBarsRequest(symbol_or_symbols=syms[i:i + chunk], timeframe=TimeFrame(1, TimeFrameUnit.Minute),
                               start=start, end=end, limit=None, feed=client._feed)
        for attempt in range(3):
            try:
                out.update(client.hist_data.get_stock_bars(req).data)
                break
            except Exception as e:
                print("   retry", attempt + 1, e, flush=True)
                time.sleep(5 * (attempt + 1))
    return out


def load_daily(client, syms, first, last):
    """Daily bars for the whole universe, cached on disk (compact: [ymd, o, h, l, c, v])."""
    CACHE.mkdir(exist_ok=True)
    p = CACHE / f"daily_{first}_{last}.json.gz"
    if p.exists():
        return json.load(gzip.open(p, "rt"))
    start = datetime.fromisoformat(first).replace(tzinfo=ET) - timedelta(days=45)
    end = datetime.fromisoformat(last).replace(tzinfo=ET) + timedelta(days=1)
    out = {}
    for i in range(0, len(syms), 200):
        got = client.get_daily_bars_bulk(syms[i:i + 200], start, end, limit=600, chunk_size=200)
        for s, bl in got.items():
            out[s] = [[b["t"].astimezone(ET).date().isoformat(), b["o"], b["h"], b["l"], b["c"], b["v"]] for b in bl]
        print(f"   daily bars {min(i + 200, len(syms))}/{len(syms)}", flush=True)
    with gzip.open(p, "wt") as f:
        json.dump(out, f)
    return out


def rebuild_list(client, day, daily, sessions, ucfg):
    d = datetime.fromisoformat(day).replace(tzinfo=ET)
    prev = sessions[sessions.index(day) - 1]
    cand = []
    for s, bl in daily.items():
        hist = [b for b in bl if b[0] < day]
        if len(hist) < 2 or hist[-1][0] != prev:
            continue
        if hist[-1][5] < ucfg["min_avg_daily_volume"]:
            continue
        if not (ucfg["price_min"] * 0.5 <= hist[-1][4] <= ucfg["price_max"] * 2):
            continue
        cand.append(s)
    pm = minute_bars(client, cand, d.replace(hour=3, minute=30), d.replace(hour=9, minute=28), chunk=100)
    scored = []
    for s in cand:
        bars = [{"t": b.timestamp, "o": float(b.open), "h": float(b.high), "l": float(b.low),
                 "c": float(b.close), "v": float(b.volume)} for b in pm.get(s, [])]
        if len(bars) < 3 or not (ucfg["price_min"] <= bars[-1]["c"] <= ucfg["price_max"]):
            continue
        hist = [b for b in daily[s] if b[0] < day]
        ph, pc, pv = hist[-1][2], hist[-1][4], hist[-1][5]
        h20 = max(b[2] for b in hist[-20:])
        res = scanner.score_breakout_candidate(s, bars, max(b["h"] for b in bars), min(b["l"] for b in bars),
                                               pv, pc, ph, h20)
        res["metrics"]["premarket_high"] = max(b["h"] for b in bars)
        res["_prev"] = (ph, h20, pv)
        scored.append(res)
    scored.sort(key=lambda r: r["candidate_score"], reverse=True)
    return cand, scored, scored[:30]


def build_day(client, day, daily, sessions, ucfg):
    d = datetime.fromisoformat(day).replace(tzinfo=ET)
    cand, scored, top = rebuild_list(client, day, daily, sessions, ucfg)
    syms = [r["symbol"] for r in top]
    reg = minute_bars(client, syms + ["SPY", "IWM"], d.replace(hour=9, minute=30), d.replace(hour=16))
    data = {}
    for r in top:
        s = r["symbol"]
        bl = [[int(b.timestamp.timestamp()), float(b.open), float(b.high), float(b.low), float(b.close),
               float(b.volume)] for b in reg.get(s, [])]
        hist = [{"t": datetime.fromisoformat(b[0]).replace(tzinfo=ET), "o": b[1], "h": b[2], "l": b[3], "c": b[4], "v": b[5]}
                for b in daily[s] if b[0] < day]
        if len(bl) < 10:
            continue
        data[s] = {"bars": bl,
                   "levels": {"premarket_high": r["metrics"]["premarket_high"], "prev_day_high": r["_prev"][0],
                              "range_20d_high": r["_prev"][1], "daily_atr": volatility.daily_atr(hist),
                              **volatility.five_day_reference(hist)},
                   "baseline": r["_prev"][2], "score": r["candidate_score"]}
    kept = [s for s in syms if s in data]
    with gzip.open(BARS / f"bars_{day}.json.gz", "wt") as f:
        json.dump({"date": day, "top30_current": kept, "top30_alt": kept, "symbols": data,
                   "_note": "[history] 9:28 list rebuilt with scanner.score_breakout_candidate (build_history.py)"}, f)
    bench = {b: [[int(x.timestamp.timestamp()), float(x.open), float(x.high), float(x.low), float(x.close),
                  float(x.volume)] for x in reg.get(b, [])] for b in ("SPY", "IWM")}
    with gzip.open(BARS / f"bench_{day}.json.gz", "wt") as f:
        json.dump(bench, f)
    (CANDS / f"{day}.json").write_text(json.dumps(
        [{"symbol": r["symbol"], "metrics": {"premarket_high": r["metrics"]["premarket_high"],
                                             "previous_day_high": r["_prev"][0], "range_20d_high": r["_prev"][1],
                                             "avg_daily_volume": r["_prev"][2]}} for r in top if r["symbol"] in data]))
    reference.build(date.fromisoformat(day), kept, verbose=False)
    return len(cand), len(scored), len(kept)


def main(first, last):
    if market_hours_now():
        print("market hours -- not starting")
        return
    client = get_client()
    ucfg = get_config()["universe"]
    t0 = time.time()
    spy = client.get_daily_bars_bulk(["SPY"], datetime.fromisoformat(first).replace(tzinfo=ET) - timedelta(days=45),
                                     datetime.fromisoformat(last).replace(tzinfo=ET) + timedelta(days=1), limit=600)["SPY"]
    sessions = [b["t"].astimezone(ET).date().isoformat() for b in spy]
    days = [s for s in sessions if first <= s <= last]
    print(f"{len(days)} trading days {days[0]} .. {days[-1]}", flush=True)
    syms = universe.get_universe_symbols(client)
    print(f"universe {len(syms)} symbols", flush=True)
    daily = load_daily(client, syms, first, last)
    for day in reversed(days):
        if market_hours_now():
            print("stopped: 9:15 ET", flush=True)
            break
        if (BARS / f"bars_{day}.json.gz").exists() and (CANDS / f"{day}.json").exists() \
                and (TRADE / "data" / "reference" / day).exists():
            continue
        t1 = time.time()
        try:
            n_c, n_s, n_k = build_day(client, day, daily, sessions, ucfg)
            print(f"{day}: prefilter {n_c}, scored {n_s}, kept {n_k}  [{time.time() - t1:.0f}s, total {(time.time() - t0) / 60:.0f} min]", flush=True)
        except Exception as e:
            print(f"{day}: FAILED {e}", flush=True)
    print("HISTORY_DONE", flush=True)


if __name__ == "__main__":
    main(sys.argv[1], sys.argv[2])