"""
simulate_prediction_pipeline.py

[SIMULATION-ONLY 2026-09-05] End-to-end offline replay that wires
together every module built this session -- stream_features ->
trend_engine -> structure_engine -> regime_engine -> setup_engine ->
prediction_engine -> entry_score -> opportunity_ranker -- against a
day's snapshot (data/snapshots/<date>/), and feeds every prediction
through outcome_engine.py (section 33) to measure how well the
deterministic next-move heuristic actually matched what happened.

This is the first time any of these modules have been run against real
data. Nothing here touches the live bot or its config's `enabled` flags
-- it imports the modules directly and calls them with cfg=None so each
one falls back to its own config.json block (all still `enabled: false`,
which nothing in this script checks or needs -- these are library
functions, not gated live code paths).

WHAT THIS DOES:
  1. Replays each symbol's regular-session bars minute by minute,
     computing the full engine chain and recording every prediction via
     outcome_engine.record_prediction().
  2. Resolves each recorded prediction against that same symbol's real
     forward bars (1m/3m/5m later) -- offline replay means the "future"
     already exists in the snapshot, so every prediction gets resolved
     (except the last few minutes of the scan window, which don't have
     a full 5m of forward data).
  3. Reports outcome_engine's calibration summary: does the engine's
     next-move distribution and expected-move estimate actually track
     reality, at all -- the honest first answer to prediction_engine.py's
     own docstring admission that "nothing here has been validated
     against real forward outcomes yet."
  4. Separately, as a secondary/entry-side check: finds each symbol's
     first entry_score.ACTION_READY bar within the scan window and
     simulates a trade, reusing simulate_regime_strategies.py's own
     ATR-trailing-stop exit engine so the P/L is directly comparable to
     that script's baseline/regime numbers and to the day's real trades.

WHAT THIS IS NOT / KNOWN GAPS (documented, not hidden -- same discipline
as simulate_regime_strategies.py's own docstring):
  - bars_sub is always [] -- the snapshot's price CSVs are Yahoo 1-minute
    bars only, no real sub-minute tick data exists to replay. Every
    sub_bucket_seconds-based feature (velocity_sub, slope_sub, trade_flow,
    pressure) is therefore always None/missing for this backtest, exactly
    as compute_features()'s own "missing" list documents rather than
    fabricating fake ticks to fill the gap. prediction_engine.py's
    momentum/acceleration/pressure components degrade gracefully to their
    neutral defaults when these are absent (see each component's `is not
    None else 0.5`-style fallback) -- scores are still produced, just
    without the finest-grained signal this session's stream.py work added.
  - quote is always None (no bid/ask in this data) -- spread checks are
    neutral/skipped everywhere, same as simulate_regime_strategies.py.
  - Entry-side backtest (item 4) simulates each symbol independently, no
    cross-symbol trading.max_positions cap or shared account risk --
    same documented simplification simulate_regime_strategies.py already
    makes. opportunity_ranker IS exercised, but only descriptively: at
    each minute where 2+ symbols are simultaneously READY, this script
    records how many symbols opportunity_ranker.rank_opportunities()
    would have had to choose between and who it would have picked --
    it does not enforce that choice by blocking the other symbols from
    also "entering" in this replay. A true portfolio-level slot
    simulation is future work, not attempted here.
  - Scans only the first SCAN_WINDOW_MINUTES of the regular session, same
    bound and same justification as simulate_regime_strategies.py
    (2026-09-04's real entries and confirmations all landed within the
    first 61 minutes) -- keeps runtime tractable and keeps this
    backtest's calibration sample concentrated in the window that
    actually matters for entries, at the documented cost of not
    calibrating the engine's behavior on quiet mid-day/afternoon action.
  - Position sizing is the same fixed $1000 notional per trade as
    simulate_regime_strategies.py, for the same reason (comparable
    aggregate dollars across many symbols without simulating account
    equity/exposure state).

Usage:
    python3 simulate_prediction_pipeline.py [YYYY-MM-DD]
"""

import csv
import json
import logging
import sys
from pathlib import Path

from config_loader import get_config
from regime_detector import bars_from_price_rows
from stream_features import compute_features
from trend_engine import classify_trend
from structure_engine import analyze_structure
from regime_engine import classify_regime
from setup_engine import evaluate_setups
from prediction_engine import compute_prediction
from entry_score import compute_entry_score, ACTION_READY, ACTION_CONFIRMING, ACTION_WAIT, DEFAULT_CONFIG as ENTRY_SCORE_DEFAULTS
from opportunity_ranker import rank_opportunities, best_opportunities
from outcome_engine import record_prediction, resolve_outcomes, summarize_calibration
from simulate_regime_strategies import simulate_exit, NOTIONAL_PER_TRADE, EARLY_BASELINE_BARS

# See simulate_regime_strategies.py's identical suppression -- this
# backtest calls into modules that share the "sip_bot" logger tree and
# would otherwise flood today's real production log with simulated lines.
logging.getLogger("sip_bot").setLevel(logging.CRITICAL + 1)

BASE_DIR = Path(__file__).resolve().parent
SCAN_WINDOW_MINUTES = 150       # see module docstring
HORIZON_BARS = {"1m": 1, "3m": 3, "5m": 5}
WARMUP_BARS = 6                  # smallest window every engine can produce a non-insufficient reading from


def load_snapshot(date: str):
    snap_dir = BASE_DIR / "data" / "snapshots" / date
    if not snap_dir.exists():
        raise FileNotFoundError(f"No snapshot at {snap_dir} -- run build_snapshot.py first.")
    trades = {}
    trades_path = snap_dir / "trades.jsonl"
    if trades_path.exists():
        for line in trades_path.read_text().splitlines():
            line = line.strip()
            if line:
                t = json.loads(line)
                trades[t["symbol"]] = t
    return snap_dir, trades


def split_premarket_regular(bars: list):
    pm = [b for b in bars if b["t"].time().strftime("%H:%M") < "09:30"]
    regular = [b for b in bars if "09:30" <= b["t"].time().strftime("%H:%M") <= "16:00"]
    return pm, regular


def _volume_baseline(regular_bars: list, i: int):
    """Same proxy-baseline reasoning as simulate_regime_strategies.py's
    _volume_baselines() -- real premarket volume isn't available in this
    data (Yahoo reports it as a flat 0), so relative_volume is proxied
    from the symbol's own early-session pace."""
    early = regular_bars[:EARLY_BASELINE_BARS]
    early_avg = (sum(b["v"] for b in early) / len(early)) if early else (regular_bars[i]["v"] or 1.0)
    early_avg = early_avg or 1.0
    return early_avg * (i + 1)


def replay_symbol(symbol: str, regular_bars: list):
    """
    Walks this symbol's regular-session bars up to SCAN_WINDOW_MINUTES,
    computing the full engine chain at each bar and recording/resolving
    predictions. Returns (readings_by_bar, resolved_records, entry_result).

    readings_by_bar: {bar_index: {"t":..., "trend":..., "structure":...,
        "regime":..., "prediction":..., "entry":...}} -- entry_score's
        EntryScoreReading (with .action) at every replayed bar, kept for
        the cross-symbol READY-contention check in run().
    entry_result: the first-READY-bar trade simulation, same shape as
        simulate_regime_strategies.py's run_one_ruleset() output.
    """
    limit = min(len(regular_bars), SCAN_WINDOW_MINUTES)
    if limit <= WARMUP_BARS:
        return {}, [], {"entered": False}

    readings_by_bar = {}
    pending_records = []  # (bar_index, PredictionRecord)
    prior_features = None
    prior_prediction = None
    confirmation_state = {}
    entry_result = {"entered": False}

    for i in range(WARMUP_BARS, limit):
        live_bars = regular_bars[:i + 1]
        as_of = live_bars[-1]["t"]
        session_elapsed_minutes = (as_of - regular_bars[0]["t"]).total_seconds() / 60.0
        avg_vol_baseline = _volume_baseline(regular_bars, i)

        features = compute_features(symbol, live_bars, bars_sub=[], quote=None,
                                     avg_vol_baseline=avg_vol_baseline, prior=prior_features,
                                     sub_bucket_seconds=30, as_of=as_of)
        trend = classify_trend(features)
        structure = analyze_structure(symbol, live_bars)
        regime = classify_regime(symbol, features, trend, structure, session_elapsed_minutes)
        setups = evaluate_setups(features, trend, structure, live_bars)
        prediction = compute_prediction(symbol, features, trend, structure, regime=regime,
                                         setups=setups, prior=prior_prediction)
        entry_reading = compute_entry_score(symbol, features, trend, structure, prediction,
                                             regime=regime, setups=setups,
                                             confirmation_state=confirmation_state, as_of=as_of)
        confirmation_state = entry_reading.confirmation

        readings_by_bar[i] = {"t": as_of, "features": features, "trend": trend, "structure": structure,
                               "regime": regime, "prediction": prediction, "entry": entry_reading}

        record = record_prediction(symbol, as_of, features.price.get("last"), prediction,
                                    context={"trend": trend.state, "structure": structure.structure,
                                             "regime": regime.regime})
        if record is not None:
            pending_records.append((i, record))

        if not entry_result["entered"] and entry_reading.action == ACTION_READY:
            entry_price = live_bars[-1]["c"]
            exit_idx, exit_price, exit_reason = simulate_exit(i, entry_price, regular_bars)
            shares = NOTIONAL_PER_TRADE / entry_price
            entry_result = {
                "entered": True, "entry_bar": i,
                "entry_time": as_of.strftime("%H:%M:%S"), "entry_price": round(entry_price, 4),
                "exit_time": regular_bars[exit_idx]["t"].strftime("%H:%M:%S"),
                "exit_price": round(exit_price, 4), "exit_reason": exit_reason,
                "trend_score": trend.score, "structure_score": structure.score,
                "prediction_score": prediction.score,
                "pl": round((exit_price - entry_price) * shares, 2),
            }

        prior_features, prior_prediction = features, prediction

    resolved = []
    for i, record in pending_records:
        forward = {}
        for horizon, n_bars in HORIZON_BARS.items():
            window = regular_bars[i + 1:i + 1 + n_bars]
            if len(window) == n_bars:      # only resolve horizons with a FULL forward window -- no partial credit
                forward[horizon] = window
        if forward:
            resolved.append(resolve_outcomes(record, forward))

    return readings_by_bar, resolved, entry_result


ACTION_RANK = {ACTION_WAIT: 0, ACTION_CONFIRMING: 1, ACTION_READY: 2}
_ENTRY_SCORE_CFG = {**ENTRY_SCORE_DEFAULTS, **get_config().get("entry_score", {})}


def _bottleneck_counts(readings_by_bar: dict) -> dict:
    """
    For every replayed bar, checks which of the 5 entry_score gates was
    below its own configured minimum (a bar can fail more than one at
    once -- all are counted, not just the first). Returns
    {criterion: count of bars where it was the blocker} plus
    bars_evaluated/bars_thresholds_cleared, so a symbol's outcome is
    explainable ("blocked on prediction_score in 61/70 bars") rather than
    a bare not-entered.
    """
    cfg = _ENTRY_SCORE_CFG
    counts = {"trend_score": 0, "structure_score": 0, "prediction_score": 0,
              "entry_score": 0, "risk_score": 0}
    cleared = 0
    for r in readings_by_bar.values():
        e = r["entry"]
        if e.insufficient_data:
            continue
        if e.thresholds_cleared:
            cleared += 1
            continue
        if e.trend_score < cfg["min_trend_score"]:
            counts["trend_score"] += 1
        if e.structure_score < cfg["min_structure_score"]:
            counts["structure_score"] += 1
        if e.prediction_score < cfg["min_prediction_score"]:
            counts["prediction_score"] += 1
        if e.entry_score < cfg["min_entry_score"]:
            counts["entry_score"] += 1
        if e.risk_score < cfg["min_risk_score"]:
            counts["risk_score"] += 1
    return counts, cleared


def summarize_symbol(sym: str, readings_by_bar: dict, entry_result: dict) -> dict:
    if not readings_by_bar:
        return {"symbol": sym, "bars_evaluated": 0, "best_action": None, "entered": False}

    best_bar, best_r = max(
        readings_by_bar.items(),
        key=lambda kv: (ACTION_RANK.get(kv[1]["entry"].action, -1),
                         kv[1]["entry"].trend_score + kv[1]["entry"].structure_score
                         + kv[1]["entry"].prediction_score + kv[1]["entry"].entry_score
                         + kv[1]["entry"].risk_score))
    best_entry = best_r["entry"]
    blocking, cleared = _bottleneck_counts(readings_by_bar)
    bottleneck = max(blocking, key=blocking.get) if any(blocking.values()) else None

    return {
        "symbol": sym,
        "bars_evaluated": len(readings_by_bar),
        "bars_thresholds_cleared": cleared,
        "best_action": best_entry.action,
        "best_bar_time": best_r["t"].strftime("%H:%M:%S"),
        "best_scores": {"trend": best_entry.trend_score, "structure": best_entry.structure_score,
                         "prediction": best_entry.prediction_score, "entry": best_entry.entry_score,
                         "risk": best_entry.risk_score},
        "primary_bottleneck": bottleneck,
        "blocking_counts": blocking,
        "entered": entry_result["entered"],
        "pl": entry_result.get("pl"),
    }


def run(date: str):
    snap_dir, real_trades = load_snapshot(date)
    prices_dir = snap_dir / "prices"
    symbols = sorted(p.stem for p in prices_dir.glob("*.csv"))

    all_records = []
    entry_results = {}
    symbol_summaries = []
    skipped_symbols = []
    ready_by_minute = {}   # {"HH:MM:SS": [symbol, ...]} -- for the cross-symbol contention check

    for sym in symbols:
        with open(prices_dir / f"{sym}.csv") as f:
            rows = list(csv.DictReader(f))
        bars = bars_from_price_rows(rows)
        _pm_bars, regular_bars = split_premarket_regular(bars)
        if len(regular_bars) <= WARMUP_BARS:
            skipped_symbols.append({"symbol": sym, "reason": f"only {len(regular_bars)} regular-session bars"})
            continue

        readings_by_bar, resolved, entry_result = replay_symbol(sym, regular_bars)
        all_records.extend(resolved)
        entry_results[sym] = entry_result
        symbol_summaries.append(summarize_symbol(sym, readings_by_bar, entry_result))

        for r in readings_by_bar.values():
            if r["entry"].action == ACTION_READY:
                ready_by_minute.setdefault(r["t"].strftime("%H:%M"), []).append(
                    {"symbol": sym, "features": r["features"], "trend": r["trend"],
                     "structure": r["structure"], "prediction": r["prediction"], "entry": r["entry"]})

    # ---- cross-symbol contention: minutes where 2+ symbols were READY at once ----
    contention_minutes = {m: rows for m, rows in ready_by_minute.items() if len(rows) >= 2}
    contention_examples = []
    for minute, rows in sorted(contention_minutes.items())[:10]:
        candidates = {row["symbol"]: {"features": row["features"], "trend": row["trend"],
                                       "structure": row["structure"], "prediction": row["prediction"],
                                       "entry": row["entry"]} for row in rows}
        ranked = rank_opportunities(candidates)
        picked = best_opportunities(ranked, max_slots=get_config().get("trading", {}).get("max_positions", 5))
        contention_examples.append({
            "minute": minute, "symbols_ready": sorted(candidates.keys()),
            "ranked": [{"symbol": c.symbol, "opportunity_score": c.opportunity_score} for c in ranked],
            "picked": [c.symbol for c in picked],
        })

    # ---- calibration summary (the headline result) ----
    calibration = summarize_calibration(all_records)

    # ---- entry-side backtest summary ----
    entered = [(sym, r) for sym, r in entry_results.items() if r["entered"]]
    total_pl = sum(r["pl"] for _, r in entered)
    wins = sum(1 for _, r in entered if r["pl"] > 0)

    out = {
        "date": date,
        "symbols_replayed": len(entry_results),
        "symbols_skipped": skipped_symbols,
        "predictions_recorded": len(all_records),
        "calibration": calibration,
        "cross_symbol_contention": {
            "minutes_with_2plus_ready": len(contention_minutes),
            "examples": contention_examples,
        },
        "entry_backtest": {
            "trades_taken": len(entered), "wins": wins,
            "win_rate_pct": round(wins / len(entered) * 100, 1) if entered else None,
            "total_pl": round(total_pl, 2),
            "trades": {sym: r for sym, r in entered},
        },
        "symbol_outcomes": sorted(symbol_summaries, key=lambda s: s["symbol"]),
    }
    out_path = snap_dir / "prediction_pipeline_backtest.json"
    out_path.write_text(json.dumps(out, indent=2, default=str))

    # ---- console report ----
    print(f"Snapshot: {date}  ({out['symbols_replayed']} symbols replayed, "
          f"first {SCAN_WINDOW_MINUTES} minutes of regular session)\n")

    print(f"PREDICTIONS RECORDED: {out['predictions_recorded']} (resolved against real forward bars)\n")
    print("CALIBRATION (prediction_engine.py vs what actually happened):")
    for h in ("1m", "3m", "5m"):
        c = calibration.get(h, {})
        if not c.get("n"):
            print(f"  {h}: no resolved predictions")
            continue
        print(f"  {h}  (n={c['n']})")
        print(f"    argmax hit rate........ {c['argmax_hit_rate']*100:.1f}%  "
              f"(random guess across 5 states ~= 20%)")
        print(f"    mean Brier (bullish)... {c['mean_brier']:.4f}  (0=perfect, 0.25=coin-flip, 1=always wrong)")
        print(f"    mean abs move error.... {c['mean_abs_move_error_pct']:.4f}%")
        print(f"    realized states........ {c['realized_state_distribution']}")
        for b in c["reliability"]:
            print(f"      bin {b['bin']}: predicted={b['predicted_mean']:.3f}  "
                  f"realized_bullish_rate={b['realized_bullish_rate']:.3f}  (n={b['n']})")

    print(f"\nCROSS-SYMBOL CONTENTION: {out['cross_symbol_contention']['minutes_with_2plus_ready']} "
          f"minutes had 2+ symbols simultaneously READY (opportunity_ranker would have had to choose)")
    for ex in contention_examples[:5]:
        print(f"  {ex['minute']}  ready={ex['symbols_ready']}  picked={ex['picked']}")

    print(f"\nENTRY-SIDE BACKTEST (entry_score.ACTION_READY trigger, ATR trailing-stop exit, "
          f"${NOTIONAL_PER_TRADE:.0f} notional/trade):")
    print(f"  trades taken: {len(entered)}   wins: {wins}   "
          f"win rate: {out['entry_backtest']['win_rate_pct']}%   total P/L: ${total_pl:.2f}"
          if entered else "  trades taken: 0")

    print(f"\nReal {date} trades vs this pipeline's entry-side replay (sanity check):")
    for sym, rt in real_trades.items():
        r = entry_results.get(sym, {"entered": False})
        sim_status = (f"entered {r['entry_time']} @ ${r['entry_price']} "
                       f"(pred_score={r['prediction_score']:.1f}) -> P/L ${r['pl']:+.2f}"
                       if r["entered"] else "NOT entered by this pipeline")
        print(f"  {sym:6} real: entry {rt['entry_time'][11:19]} @ ${rt['entry_price']:.2f} "
              f"P/L ${rt['current_pl']:+.2f}   |   pipeline: {sim_status}")

    if skipped_symbols:
        print(f"\nSKIPPED ({len(skipped_symbols)}): " +
              ", ".join(f"{s['symbol']} ({s['reason']})" for s in skipped_symbols))

    print(f"\nPER-SYMBOL OUTCOME (all {len(symbol_summaries)} replayed symbols, "
          f"best bar reached each):")
    print(f"  {'SYMBOL':7}{'BEST ACTION':12}{'T':>6}{'S':>6}{'P':>6}{'E':>6}{'R':>6}"
          f"  {'CLEARED/BARS':13}  {'BOTTLENECK':16}  RESULT")
    for s in sorted(symbol_summaries, key=lambda x: x["symbol"]):
        if s["bars_evaluated"] == 0:
            continue
        bs = s["best_scores"]
        result = f"ENTERED pl=${s['pl']:+.2f}" if s["entered"] else "-"
        print(f"  {s['symbol']:7}{s['best_action']:12}"
              f"{bs['trend']:6.1f}{bs['structure']:6.1f}{bs['prediction']:6.1f}"
              f"{bs['entry']:6.1f}{bs['risk']:6.1f}"
              f"  {s['bars_thresholds_cleared']:>4}/{s['bars_evaluated']:<7}"
              f"  {str(s['primary_bottleneck']):16}  {result}")

    print(f"\nWrote {out_path.relative_to(BASE_DIR)}")


if __name__ == "__main__":
    date = sys.argv[1] if len(sys.argv) > 1 else "2026-09-04"
    run(date)