o LjV@sdZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z m Z m Z ddl mZddlmZeejZejdeeddlmZded id <ddlZdd lmZddlZe jZ eed d Z!dedede fddZ"d2ddZ#defddZ$defddZ%GdddZ&ddZ'e d d!Z(d3d#d$Z)Gd%d&d&Z*Gd'd(d(Z+d4d)d*Z,d+d,Z-d-d.Z.d/d0Z/e0d1kre/dSdS)5a simulate.py -- trade1 backtester. Replays saved trade-by-trade market data through the REAL bot: it builds monitor.SessionOrchestrator (the same class that trades live) with a replay clock, a simulated broker and a replayed data stream, so every buy/sell decision comes from the same code path and the same rule files (breakout_rules.py, reversal_rules.py, exit_rules.py) as live trading. python simulate.py --days all every saved day python simulate.py --date 2026-09-22 one day python simulate.py --days all --entry breakout_rules --exit exit_rules python simulate.py --days all --rules-dir /some/dir --entry my_test_rules (--rules-dir: also import rule files from that folder -- for trying a variant without touching the real rule files) python simulate.py --fixed-entries sim/entries/j_20day.json --exit exit_rules EXIT test: buys exactly the listed entries (sim/rules/fixed_entries.py), only on their days and symbols, no slot limit -- compare exits on the same entries python simulate.py ... --set exit_rules.swing_minutes=3 --set exit_rules.trigger=touch override any config value for this run only (value parsed as JSON) Data: candidates sim/candidates/.json (the 9:28 top-30 lists; 8/27-9/24 are the rebuilt lists from the 20-day study, 9/25 the bot's real list), else data/candidates/_scanner.json ticks simulator.cache_dir in config.json (shared with screener/trade: //.jsonl); missing symbols are downloaded from Alpaca on first use Output: data/simulations/.json every trade + per-day summary data/simulations//decisions_.jsonl rule decisions (on change) logs/sim_*.log (kept apart from the live bot's log) Simplifications (same as screener/trade's simulator): fills at the last trade price (no slippage), fixed equity for sizing, no intraday rescans (each day's 9:28 list is watched all day), 5-second poll grid. N)ThreadPoolExecutor as_completed)datetime timedeltatimezone)Path)ZoneInfo) get_configsimloggingfilename_prefix SymbolBufferschedulerdate_strhhmmssreturnc Cs@dd|dD\}}}t|}t|j|j|j|||tdS)Ncss|]}t|VqdSN)int.0xr$/var/www/screener/trade1/simulate.py Dz_et_dt..:)tzinfo)splitr fromisoformatyearmonthdayET)rrhmsdrrr_et_dtCs r(ccs@|}||krt|t|d|}||fV|}||ksdSdS)Nminutes)minr)startendr+curnxtrrr_chunksLs r1pathc Cs|d}d}t|d\}t||D]N\}} |||| } |||| } dd| Ddd| D} | jddd | D]\} }}}|t| |||gd q<|t | 7}~ ~ ~ t qWd n1skwY| ||S) z6One 20-min chunk at a time -- this host has ~2 GB RAM.z .jsonl.tmprwcS$g|]}|dd|d|dfqS)ttradepr&rrr5rrr \$z!_cache_symbol..cSr4)r5quotebar)rqrrrr9]r:cSs|dSNrr)errr^z_cache_symbol..)key N) with_suffixopenr1get_historical_tradesget_historical_quotessortwritejsondumps isoformatlengccollectrename)clientsymbolr-r.r2tmpnfr=r<tradesquotesevtskindryrrr _cache_symbolTs&    "   r] cache_dirc s<jdddfdd|D}|sdStd}t||dtt||dt|tdt|d tjd t d d Rfd d|D}t |D]7}ztd||d| dtjd WqTt y}ztd||d|tjd WYd}~qTd}~wwWddS1swYdS)NTparentsexist_okcs"g|] }|ds|qS.jsonl)existsrr&r^rrr9js"z ensure_cache..rmarket_open_timemarket_close_timez downloading ticks for z symbols ...file) max_workersc s,i|]}t||d|qSrb)submitr]rer^rRr.poolr-rr ss,z ensure_cache.. : z eventsz : FAILED ) mkdirr r( astimezoneUTCprintrNsysstderrrrresult Exception) client_fnsymbolsrr^todoschedfutsfutr@rrnr ensure_cachehs(   (*"rc@s6eZdZdZdefddZddZddZd d Zd S) EventReaderz/Streams one symbol's cached ticks line by line.r2cCs(|rt|nd|_d|_|dSr)rdrF_f_next_advance)selfr2rrr__init__~s zEventReader.__init__cCsL|jr|jnd}|sd|_dSt|\}}}}t||||f|_dS)N)rreadlinerrKloadsrr)rlinerZr[r=r<rrrrs zEventReader._advancecCsn|jdur1|jd|kr3|j\}}}}|dkr|n||||||jdur5|jd|ks dSdSdSdS)Nrr6)rr)rr5on_tradeon_quoterZr[r=r<rrr drain_up_tos (zEventReader.drain_up_tocCs|jr |jdSdSr)rcloserrrrrszEventReader.closeN) __name__ __module__ __qualname____doc__rrrrrrrrrr{s  rc st|dti}ddtjfddtjffD]`\}}}||d}|r-t| nifdd|D} | rr|sr| | t |d t d d } | D]\} } |fd d| D} | ri| | <qT| t||<q|S) zDaily ATR and 5-day reference as of the morning of date_str (cached next to the ticks, same files screener/trade's simulator uses).z00:00:00 daily_atr-ref5d.jsoncg|]}|vr|qSrrre)datarrr9sz_daily_ref..)dayssecondscsg|] }|dkr|qS)r5rrr<)r"rrr9)r(rtru volatilityrfive_day_referencerdrKr read_textget_daily_bars_bulkritems write_textrL)r{r|rr^outname days_backfnr7missingbarsr&blvr)rr"r _daily_refs$   $ rrbars1mc Csi}|D]=}||d|d}|s3t|d}|j||t|ddd} |tdd| Dd dt|D||<q|S) aAlpaca's OFFICIAL 1-min bars (what the live bar stream delivers), cached next to the ticks. Used for the watched symbols (the live stream upserts them into the buffer and forwards them to the rule modules) and for config streaming.benchmark_symbols (SPY/IWM, bars only)._rz09:30:00z16:00:00i)r-r.limitc Ss@g|]}|jt|jt|jt|jt|jt|jgqSr) timestamprMfloatrFhighlowrvolumerrrrr9s  z _minute_bars..c Ss<g|]}t|d|d|d|d|d|ddqS)rrrrk)r5or$lcr)rr)rrrrrr9s4)rdr(get_minute_barsrrKrLrr) r{r|rr^prefixrr&r7r=rawrrr _minute_barss    rc@sVeZdZddZddZddZddZd d Zd d ZdddZ ddZ ddZ d S)SimPositionManagercCs(||_||_||_||_i|_g|_dSr)equitycfg max_positionsclock positionsrW)rr trading_cfgrrrrrrs  zSimPositionManager.__init__cCst|j|jkSr)rNrrrrrrhas_available_slotsz%SimPositionManager.has_available_slotcCs ||jvSr)r)rrSrrris_symbol_open z!SimPositionManager.is_symbol_opencC t|jSr)listrrrrrget_open_symbolsrz#SimPositionManager.get_open_symbolscCrr)rrWrrrrclosed_trades_todayrz&SimPositionManager.closed_trades_todaycCsl||}|dks |s dSt|j|jdd|}t|t|j|jdd|}||jdkr4|SdS)Nraccount_risk_pct_per_tradegY@#max_position_notional_pct_of_equity min_shares)rrrr,)r entry_price stop_pricerisk_per_shareqtyrrr calculate_qtys  "z SimPositionManager.calculate_qtyNc Csb||s |s dS|||}|dkrdS||||||p#g||p'i||d |j|<dS)NFr) rSrrr entry_timereasonssetupplan peak_price low_priceT)rrrrrMr)rrSrrrrrrrrrenter_positions   z!SimPositionManager.enter_positioncCs<|j|}|rt|d||d<t|d||d<dSdS)Nrr)rgetmaxr,)rrSpricer7rrrmarks zSimPositionManager.markcCs|j|d}|s dSt|d|t|d|}}||d|d}|ji||||d||t||ddddt|d t||d|dd t||d|d|d d dS) Nrrrrclosedrdrkr) exit_price exit_time exit_reasonstatusrrpl_pct pl_dollars mfe_dollarsgiveback_dollars) rpoprr,rWappendrrMround)rrSrreasonr7peakrplrrr exit_positions z SimPositionManager.exit_position)NNN) rrrrrrrrrrrrrrrrrs  rc@szeZdZdZddZdddZddZd d Zd efd d Z ddZ ddZ ddZ dddZ ddZddZddZdS) SimStreamzStand-in for stream.StreamManager: the same SymbolBuffer class the live stream uses, fed from saved ticks; forwards ticks and completed 1-min bars to listeners exactly like the live stream does.cstdi}|ddtdt|dddfdd |D|_g|_t|_t||_i|_ i|_ i|_ i|_ dS) N streamingsub_minute_bucket_secondsrsub_minute_buffer_minutesr<csi|] }|tdqS))sub_bucket_seconds sub_maxlenr resub_nsub_srrrp sz&SimStream.__init__..) r rrrbuffers listenersset bars_only _subscribed _last_barbenchofficial _released)rr|scrrrrs   zSimStream.__init__Nc Cs|j|d}|t|kr\||dtddt|kr\||}|dur:||d|d|d|d|d |d|d |t||d7}|t|kr\||dtddt|ks||j|<dS) Nrr5rr*rr$rrron_bar)r rrNr BAR_DELAYr _notifydict)rr&rr5bufir<rrr_releases*,*zSimStream._releasecCs&|jD] \}}||||qdSr)rrr)rr5r&rrrr feed_benchszSimStream.feed_benchcGs(|jD]}t||d}|r||qdSr)rgetattr)rmethodargslstrrrrr #s  zSimStream._notifyreadercsVjfdd}fdd}||||jg|dS)Ncs$|||d|||dS)Nr)rr )rZrsizerrrSrrr,sz SimStream.feed..on_tradec s(|||d|||dddS)Nrg)rr )rZbidaskrrrr0sz SimStream.feed..on_quote)rrrrr)rrSrr5rrrrrfeed)s zSimStream.feedcCsJ|j|}|dur||jvr|j|d|j|dS|r#|SgSr?)rrrr get_barsrrSr<rrrr8s zSimStream.get_barscCs|j|}|r |SgSr)rr get_bars_subrrrrr>s zSimStream.get_bars_subcCs|j|}|r |jSdSr)rr latest_quoterrrr get_quoteBs zSimStream.get_quote4@cCs|j|}|r ||SdSr)rrget_trade_imbalance)rrSwindow_secondsr<rrrr#Fs zSimStream.get_trade_imbalancecCdSrrrr|rrr add_symbolsJzSimStream.add_symbolscCr%rrr&rrrremove_symbolsMr(zSimStream.remove_symbolscCr%rrrrrrstopPr(zSimStream.stopr)r")rrrrrrrr rrrrr!r#r'r)r*rrrrrs    rcCsp|rt|gngtdd|dtdd|dgD]}|r0t||fSqtd|)Nr candidatesrrz _scanner.jsonzno candidate list for )rBASE_DIRrdrKrrFileNotFoundError)roverrider7rrrload_candidatesUs r/cCs tddtdddDS)Ncss|]}|jVqdSr)stem)rr7rrrr_szall_days..r r+z*.json)sortedr,globrrrrall_days^s r3csRt}|d}t||j\}}|jdur5|j|tfdd|Dddtdd|DD}dd|Dt|did td d |t ||t ||} d t ||d  t ifdd} t||d|jp{|dd| } t} t|fdd|didgD|dd| _t||| _|d|d} t| dfdd}tjt| | | |dd}|jD]}|q||dd|D| d | d!d"fd#dD}d }t ||d$ t }t ||d% t }t|d&d'}||krZ|d <| |D] }| ||||q| D]}| |}|r=| ||d(d)q)|!| "rQ||krQ|#||7}||ks |d <|jD]}|$d*qa| D]}| |}|%||r|d(d)n| j&|d+d,qn|'D]}|(q(~ ~t)*| j+t,|fS)-Nrcsg|] }|dvr|qSrSrrr)wantrrr9hrzrun_day..cSsg|]}|idqS))rSmetricsrrrrrr9iscSsh|]}|dqSr4rr5rrr izrun_day..cSsg|]}|dqSr4rr5rrrr9jr9 simulatorr^rcacher5rgcsdS)Nr5rr)nowrrrAprBzrun_day..tradingrcrrrr)r|rrr9ssrbenchmark_symbolsr)r decisions_rcr3cs,tjddi|tdddS)Nrr5)defaultrD)rJrKrLrMstr)rec)dec_fr<rrsinkys,zrun_day..sinkF)rR position_mgrstreamr decision_sinklivecSs i|] }|d|ddqS)rSr7avg_daily_volume)rr5rrrrp zrun_day..rr)r+ baselinesrrcs i|] }|t|dqSrb)rrerfrrrprJno_new_entries_afterforce_liquidate_timepoll_interval_secondsrron_session_endr END_OF_DAY)-r r/candidates_file _fixed_symsrrr1rr,rrr(rtrurrrrrrrFmonitorSessionOrchestratorobject all_modulesload _apply_scanrrrrrr_update_open_positionsr_scan_for_entriescall_sellrvaluesrrOrPrWrA)rrrr{out_dirrr~candscpathrefrpmrFdec_pathrDrr%readersr5cutoffeodstepr&rrr)r^rCr<r|r6rrun_daybs $  $             .  ricstjttjd}|jddd|jddd|jddd|jd d d|jd d d|jd dd|jdtdd|jdtdd|jddd|jddd|jddgddd|}|jD]8}| dd\}}zt |}Wn t y{Ynwt }| d}|dd D]}||i}q|||d <qad|_|jrt|j}tjd!ttd"d#d$gt d#d%<t|t d$id&<i|_t t|d'D]} |j| d(t| d)q|jsd*|_|js|jsd+t|j|_|j r tjd!tt|j t d#i} d,| d-<|j!r,|js,d.d/|j! d+D| d%<|j"r7|j"#| d0<|jr?|jgn|jd1krHt$n d2d/|jpPd3 d+D} | s^|%d4ifd5d6} |j&pqt| 'j&} |j(pd7d8| )d%gd9t| )d0d:t*+,d;}td<d=|}|j-d>d>d?t.d@| )d%dA| )d0dB| dCdDt/| dE| tj0dF|| )d%| )d0| idG}| D]}t11}t2||| | |\}}||dH|dI|<t3dJdK|D}t.|dLt/|dMt3dNdK|DdO|dPdQt11|dRdS d>dT|D]B}t.dU|d)dVdW|dXdY|dZd[d\d]|d^d_d`|dad[d\d]|dbd_dc|dddPde|dfddgdhd>dTqtd<d=|di4t j5|dtdjqdkd/|dI6D}dld/|D}t.dmt/| dnt/|dMt/|dot3dpdK|DdPdqt/|t/|drt3dsdK|DdPdtt3dudK|DdPdS)vN) descriptionformatter_classz--datezone day, YYYY-MM-DD)helpz--daysz='all' (every file in sim/candidates) or a comma list of datesz--candidates-filez,use this candidate list (single --date only)z--entryzAcomma list of entry modules (default: config rules.entry_modules)z--exitz/exit module (default: config rules.exit_module)z --rules-dirz&extra folder to import rule files fromz--max-positionsz%default: config trading.max_positions)typerlz--equityz;sizing equity (default: the paper account's current equity)z--namez1results name (default: from the modules and time)z--fixed-entriesz/exit test: buy exactly the entries in this filez--setrzSECTION.KEY=VALUEz1override a config value for this run (repeatable))actionr@metavarrl=r.rOrr rules fixed_entries entry_modulesrjentriesdaterSc,F hot_reloadcSg|] }|r|qSrstriprrrrr9rzmain.. exit_moduleallcSrzrr{)rr'rrrr9rrzgive --date or --dayscs&dvrddlm}|d<dS)Nrr get_client) alpaca_clientrr_clientrrr{s  zmain..client_fnsim_+__rz %Y%m%d_%H%M%Sr simulationsTr_ztrade1 simulator | entry=z exit=z | equity=$z,.2fz | z day(s) -> ri)rrtr}rr)r+rWrcs|]}|dVqdSrNrr8rrrrrzmain..rrz trades, css|] }|ddkVqdSrrNrr8rrrrsz winners, $z+.2fz [z.0fzs])flushrq6z [rz] r zZ @ rz.2fz -> rrz $rz (rr)r)indentr@cSsg|] }|dD]}|qqS)rWr)rrr5rrrr9scSsg|] }|ddkr|qS)rrrr8rrrr9rz TOTAL z days: z winners ($csrrrr8rrrrrz), z losers ($css$|] }|ddkr|dVqdSrrr8rrrrs"z), net $csrrrr8rrrrr)7argparseArgumentParserrRawDescriptionHelpFormatter add_argumentrr parse_argsrrrKr ValueErrorr setdefaultrSrsrresolverwr2insertrAr,rXrFaddrrvrjoinr1 rules_direntryexitr|r3errorr get_accountrrrr<strftimersrvrNrxtimerisumrrLr^)aprkvkrnodepartsr7fer@rcrr{rrr_resultsr't0rWsrcrr5alltwinsrrrmains      "  . & 8 ," r__main__)r))rr)1rrrO importlibrKrwrconcurrent.futuresrrrrrpathlibrzoneinfor__file__rparentr,r2rrA config_loaderr rrrFrrTutcrur#r(r1r]rrrr rrrr/r3rirrrrrrsJ&       @ R E^