"""
group_tree.py -- [2026-10-09] Step 2: readable GROUP FILTERS from group_rows.csv.gz (group_build.py).
A small regression tree (depth 4, numpy only) on the net trade P/L (+3% / -1.5% / close, minus the real 0.2% cost)
-> every leaf is one group = a short list of plain filters ("daily ATR >= 8.7% AND above VWAP >= 0.9% AND ...").
Fitted on 2021-10..2023-12 ONLY; each group is labelled from the TRAINING years (WINNER if its training average
> +0.15% per trade, LOSER if < -0.30%, else NEUTRAL) and then checked on 2024-2026, which it never saw.
Signal test: per stock-day, the FIRST checkpoint that lands in a WINNER group = the buy signal.
Writes the model for the live log-only watcher: /var/www/screener/trade/data/group_model.json
    python3 group_tree.py
"""
import csv
import gzip
import json
from collections import defaultdict
from pathlib import Path

import numpy as np

HERE = Path(__file__).resolve().parent
MODEL = Path("/var/www/screener/trade/data/group_model.json")
COST = 0.2
SPLIT = "2024-01-01"
DEPTH, MIN_LEAF = 4, 400
STYLES = ["none", "straight", "dip, then up", "deep V", "base break", "slow grind", "late move", "choppy"]
FEATS = {"t": "minute since 9:30", "gap": "gap %", "atrp": "daily ATR %", "pmv_log": "log10 premarket volume",
         "ret": "move from open %", "hi": "day high so far %", "lo": "day low so far %", "ret_atr": "move from open in ATRs",
         "vwap": "vs VWAP %", "vol_rel_log": "log10 volume so far / prior day", "r15": "last 15 min %",
         "eff15": "last 15 min straightness", "rng15": "last 15 min range %", "pos": "position in day range (0-1)",
         "off_hi": "below day high %", **{f"st_{s}": f"style = {s}" for s in STYLES}}


def load():
    c = defaultdict(list)
    with gzip.open(HERE / "group_rows.csv.gz", "rt") as fh:
        for r in csv.DictReader(fh):
            c["day"].append(r["day"]); c["sym"].append(r["sym"]); c["res"].append(r["res"])
            f = lambda k: float(r[k]) if r[k] not in ("", "nan") else np.nan
            for k in ("t", "gap", "atrp", "ret", "hi", "lo", "ret_atr", "vwap", "r15", "eff15", "rng15", "pos", "off_hi",
                      "pl", "max_after", "to_close"):
                c[k].append(f(k))
            c["pmv_log"].append(np.log10(max(f("pmv") or 0, 0) + 1))
            vr = f("vol_rel"); c["vol_rel_log"].append(np.log10(vr) if vr == vr and vr > 0 else np.nan)
            for s in STYLES:
                c[f"st_{s}"].append(1.0 if r["style"] == s else 0.0)
    D = {k: np.array(v) for k, v in c.items()}
    D["y"] = np.clip(D["pl"], -10, 10) - COST
    return D


def best_split(X, y, idx):
    base = ((y[idx] - y[idx].mean()) ** 2).sum()
    best = None
    for f, x in X.items():
        xv = x[idx]
        ok = ~np.isnan(xv)
        if ok.sum() < 2 * MIN_LEAF:
            continue
        for q in np.unique(np.nanquantile(xv, np.linspace(0.1, 0.9, 17))):
            for nan_left in (False, True):
                left = (xv < q) | (~ok & nan_left)
                nl = left.sum(); nr = len(idx) - nl
                if nl < MIN_LEAF or nr < MIN_LEAF:
                    continue
                yl, yr = y[idx][left], y[idx][~left]
                gain = base - ((yl - yl.mean()) ** 2).sum() - ((yr - yr.mean()) ** 2).sum()
                if best is None or gain > best[0]:
                    best = (gain, f, float(q), nan_left)
    return best


def grow(X, y, idx, depth, path):
    node = {"n": int(len(idx)), "mean": float(y[idx].mean())}
    s = best_split(X, y, idx) if depth < DEPTH else None
    if s is None or s[0] <= 0:
        node["leaf"] = True; node["path"] = path; return node
    _, f, q, nl = s
    xv = X[f][idx]; left = (xv < q) | (np.isnan(xv) & nl)
    node.update(f=f, q=q, nan_left=nl,
                L=grow(X, y, idx[left], depth + 1, path + [(f, "<", q)]),
                R=grow(X, y, idx[~left], depth + 1, path + [(f, ">=", q)]))
    return node


def leaves(node, out=None):
    out = [] if out is None else out
    if node.get("leaf"):
        out.append(node)
    else:
        leaves(node["L"], out); leaves(node["R"], out)
    return out


def assign(node, row):
    """row: dict feature -> value. Returns the leaf (same walk as trade/group_watch.py)."""
    while not node.get("leaf"):
        v = row.get(node["f"], np.nan)
        go_left = (v < node["q"]) if v == v else node["nan_left"]
        node = node["L"] if go_left else node["R"]
    return node


def describe(path):
    parts = []
    for f, op, q in path:
        if f.startswith("st_"):
            parts.append(("NOT " if op == "<" else "") + FEATS[f])
        else:
            parts.append(f"{FEATS[f]} {op} {q:.2f}")
    return " AND ".join(parts)


def main():
    D = load()
    X = {f: D[f] for f in FEATS}
    tr = np.where(D["day"] < SPLIT)[0]; te = np.where(D["day"] >= SPLIT)[0]
    print(f"{len(D['day'])} rows ({len(set(D['day']))} days); training {len(tr)} rows < {SPLIT}, test {len(te)} rows")
    print(f"baseline (buy at every checkpoint): train {D['y'][tr].mean():+.3f}%  test {D['y'][te].mean():+.3f}% per trade, "
          f"WIN {np.mean(D['res'][te] == 'WIN'):.0%} LOSS {np.mean(D['res'][te] == 'LOSS'):.0%}")
    tree = grow(X, D["y"], tr, 0, [])
    L = leaves(tree)
    leaf_of = np.zeros(len(D["day"]), dtype=np.int64)

    def walk(node, idx):                     # vectorised assign() (1 GB server: no per-row dicts)
        if node.get("leaf"):
            leaf_of[idx] = id(node); return
        v = X[node["f"]][idx]
        left = np.where(np.isnan(v), node["nan_left"], v < node["q"])
        walk(node["L"], idx[left]); walk(node["R"], idx[~left])
    walk(tree, np.arange(len(D["day"])))
    years = sorted({d[:4] for d in D["day"][te]})
    print("\nGROUPS (label from the training years; then the unseen test years)")
    for g, lf in enumerate(sorted(L, key=lambda n: -n["mean"]), 1):
        lf["gid"] = f"G{g}"
        lf["label"] = "WINNER" if lf["mean"] > 0.15 else ("LOSER" if lf["mean"] < -0.30 else "NEUTRAL")
        lf["desc"] = describe(lf["path"])
        k = leaf_of == id(lf); kt = k & (D["day"] >= SPLIT)
        lf["test_n"] = int(kt.sum()); lf["test_mean"] = float(D["y"][kt].mean()) if kt.sum() else None
        print(f"\n{lf['gid']} {lf['label']:8} train {lf['n']:6} rows {lf['mean']:+.3f}% | TEST {kt.sum():6} rows "
              f"{D['y'][kt].mean() if kt.sum() else 0:+.3f}% WIN {np.mean(D['res'][kt] == 'WIN') if kt.sum() else 0:.0%} "
              f"LOSS {np.mean(D['res'][kt] == 'LOSS') if kt.sum() else 0:.0%} | by year "
              + " ".join(f"{y}:{D['y'][kt & (np.char.startswith(D['day'], y))].mean():+.2f}" for y in years
                         if (kt & np.char.startswith(D["day"], y)).sum()))
        print(f"   {lf['desc']}")
    # signal test: first checkpoint per stock-day in a WINNER group
    win_ids = {id(lf) for lf in L if lf["label"] == "WINNER"}
    first = {}
    for i in np.argsort(D["t"], kind="stable"):
        key = (D["day"][i], D["sym"][i])
        if key not in first and leaf_of[i] in win_ids:
            first[key] = i
    sig = np.array(sorted(first.values()), int)
    print(f"\nBUY SIGNALS (first WINNER checkpoint per stock-day): {len(sig)}")
    for lab, part in (("train", sig[D["day"][sig] < SPLIT]), ("TEST", sig[D["day"][sig] >= SPLIT])):
        if len(part):
            print(f"  {lab}: {len(part)} signals ({len(part) / len(set(D['day'][part])):.1f}/day) | {D['y'][part].mean():+.3f}%/trade "
                  f"| WIN {np.mean(D['res'][part] == 'WIN'):.0%} LOSS {np.mean(D['res'][part] == 'LOSS'):.0%} | hold to close "
                  f"{np.nanmean(D['to_close'][part]) - COST:+.3f}% | median max gain after {np.nanmedian(D['max_after'][part]):.2f}%")
    for y in years:
        p = sig[np.char.startswith(D["day"][sig], y)]
        if len(p):
            print(f"    {y}: {len(p):5} signals {D['y'][p].mean():+.3f}%/trade  WIN {np.mean(D['res'][p] == 'WIN'):.0%}")
    by_t = defaultdict(list)
    for i in sig[D["day"][sig] >= SPLIT]:
        by_t[int(D["t"][i])].append(D["y"][i])
    print("  test signals by checkpoint: " + "  ".join(f"{9 + (30 + t) // 60}:{(30 + t) % 60:02d} n{len(v)} {np.mean(v):+.2f}%"
                                                      for t, v in sorted(by_t.items())))

    def strip(n):
        out = {k: n[k] for k in ("n", "mean") if k in n}
        if n.get("leaf"):
            out.update(leaf=True, gid=n["gid"], label=n["label"], desc=n["desc"], test_n=n["test_n"], test_mean=n["test_mean"])
        else:
            out.update(f=n["f"], q=n["q"], nan_left=bool(n["nan_left"]), L=strip(n["L"]), R=strip(n["R"]))
        return out
    MODEL.parent.mkdir(parents=True, exist_ok=True)
    json.dump({"built": "2026-10-09", "split": SPLIT, "checks": [15, 30, 45, 60, 90, 120], "win_pct": 3.0, "stop_pct": 1.5,
               "cost": COST, "tree": strip(tree)}, open(MODEL, "w"), indent=1)
    print(f"\nmodel -> {MODEL}")


if __name__ == "__main__":
    main()
