o
    Ó}¼j�#  ã                   @   sJ  d Z ddlZddlZddlZddlZddlmZmZmZ ddlm	Z	 ddl
mZ e	eƒ ¡ jZejd Zej deeƒ¡ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlZddlZddl Z ddl!Z!edƒZ"ed Z#ed d Z$ed Z%dd„ Z&ddd„Z'dd„ Z(dd„ Z)dd„ Z*dd„ Z+e,dkr£e+ej-d ej-d ƒ dS dS ) a  
build_history.py -- [2026-09-29] 6 months of test data (the user: "create the 6 month data,
so we can test it and add more patterns to the book"). For every trading day in the range,
rebuild the scanner's 9:28 top-30 the way the live scanner picks it, and save the same files
the other days have, so every study / simulator picks the days up automatically:

  bars/bars_<date>.json.gz     full-day 1-min bars (9:30-16:00) of the top 30 + levels
  bars/bench_<date>.json.gz    SPY / IWM 1-min bars
  tick20/cands/<date>.json     the list for the tick simulator (ticks download on first use)
  data/reference/<date>/       14-day reference files of the top 30 (reference.build)

List rebuild, per day (as the live bot: universe.prefilter_by_snapshot + scanner.scan):
  universe  today's tradable asset list (universe.get_universe_symbols -- CAVEAT: symbols
            delisted since are missing, so the lists lean slightly toward survivors)
  prefilter previous session volume >= min_avg_daily_volume; last premarket price
            (bars 3:30-9:28) in [price_min, price_max]
  score     scanner.score_breakout_candidate (the live function), >= 3 premarket bars
  top 30    by candidate_score
Not included: intraday rescans (the live bot adds stocks at 9:40, 10:10, ...), buy/sell
flow (needs tick data: tick20 simulator downloads it per day when it runs).

    python3 build_history.py 2026-03-02 2026-08-26      newest day first; resumable;
                                                        stops at 9:15 ET, never runs 9:15-16:05
é    N)ÚdateÚdatetimeÚ	timedelta)ÚPath)ÚZoneInfoé   )ÚStockBarsRequest)Ú	TimeFrameÚTimeFrameUnit)Ú
get_client)Ú
get_configzAmerica/New_YorkÚbarsÚtick20ÚcandsÚhistoryc                  C   s6   t  t¡} |  ¡ dk od| j| jf  kodk S   S )Né   )é	   é   )é   r   )r   ÚnowÚETÚweekdayÚhourÚminute©Út© r   ú</var/www/screener/trade/reports/open_window/build_history.pyÚmarket_hours_now3   s   
,r   é2   c           
      C   sº   i }t dt|ƒ|ƒD ]P}t|||| … tdtjƒ||d | jd�}t dƒD ]4}z| | j 	|¡j
¡ W  n% tyY }	 ztd|d |	dd� t d|d  ¡ W Y d }	~	q%d }	~	ww q
|S )	Nr   r   )Úsymbol_or_symbolsÚ	timeframeÚstartÚendÚlimitÚfeedé   z   retryT©Úflushr   )ÚrangeÚlenr   r	   r
   ÚMinuteÚ_feedÚupdateÚ	hist_dataÚget_stock_barsÚdataÚ	ExceptionÚprintÚtimeÚsleep)
ÚclientÚsymsr"   r#   ÚchunkÚoutÚiÚreqÚattemptÚer   r   r   Úminute_bars8   s    
ÿ€þ€r=   c                 C   s:  t jdd� t d|› d|› d� }| ¡ rt t |d¡¡S t |¡j	t
d�tdd	� }t |¡j	t
d�td
d	� }i }tdt|ƒdƒD ]9}| j|||d … ||ddd�}	|	 ¡ D ]\}
}dd„ |D ƒ||
< qZtdt|d t|ƒƒ› dt|ƒ› �dd� qDt |d¡�}t ||¡ W d  ƒ |S 1 s–w   Y  |S )zRDaily bars for the whole universe, cached on disk (compact: [ymd, o, h, l, c, v]).T)Úexist_okÚdaily_Ú_ú.json.gzÚrt©Útzinfoé-   ©Údaysr   r   éÈ   é  )r$   Ú
chunk_sizec              	   S   sB   g | ]}|d    t¡ ¡  ¡ |d |d |d |d |d g‘qS ©r   ÚoÚhÚlÚcÚv©Ú
astimezoner   r   Ú	isoformat©Ú.0Úbr   r   r   Ú
<listcomp>S   s   B zload_daily.<locals>.<listcomp>z   daily bars ú/r'   ÚwtN)ÚCACHEÚmkdirÚexistsÚjsonÚloadÚgzipÚopenr   ÚfromisoformatÚreplacer   r   r)   r*   Úget_daily_bars_bulkÚitemsr2   ÚminÚdump)r5   r6   ÚfirstÚlastÚpr"   r#   r8   r9   ÚgotÚsÚblÚfr   r   r   Ú
load_dailyG   s$    ,
ÿþrn   c                    s  t  ˆ ¡jtd�}|| ˆ ¡d  }g }| ¡ D ]D\}}	‡ fdd„|	D ƒ}
t|
ƒdk s3|
d d |kr4q|
d d |d	 k r?q|d
 d |
d d   krU|d d ksWn q| |¡ qt| ||jddd�|jddd�dd�}g }|D ]†}dd„ | 	|g ¡D ƒ}t|ƒdk sš|d
 |d d   kr™|d ks›n qt‡ fdd„|| D ƒ}
|
d d |
d d |
d d }}}t
dd„ |
dd … D ƒƒ}t ||t
dd„ |D ƒƒtdd„ |D ƒƒ||||¡}t
dd„ |D ƒƒ|d d< |||f|d < | |¡ qt|jd!d"„ d#d$� |||d d… fS )%NrC   r   c                    ó   g | ]
}|d  ˆ k r|‘qS ©r   r   rT   ©Údayr   r   rW   _   ó    z rebuild_list.<locals>.<listcomp>é   éÿÿÿÿr   r   Úmin_avg_daily_volumeÚ	price_ming      à?é   Ú	price_maxr&   é   ©r   r   r   é   éd   )r7   c              	   S   s>   g | ]}|j t|jƒt|jƒt|jƒt|jƒt|jƒd œ‘qS )rK   )Ú	timestampÚfloatr`   ÚhighÚlowÚcloseÚvolumerT   r   r   r   rW   j   s
    ÿ
ÿrO   c                    ro   rp   r   rT   rq   r   r   rW   n   rs   c                 s   ó   � | ]}|d  V  qdS )rt   Nr   rT   r   r   r   Ú	<genexpr>p   ó   € zrebuild_list.<locals>.<genexpr>iìÿÿÿc                 s   r„   ©rM   Nr   rT   r   r   r   r…   q   r†   c                 s   r„   )rN   Nr   rT   r   r   r   r…   q   r†   c                 s   r„   r‡   r   rT   r   r   r   r…   s   r†   ÚmetricsÚpremarket_highÚ_prevc                 S   s   | d S )NÚcandidate_scorer   )Úrr   r   r   Ú<lambda>v   s    zrebuild_list.<locals>.<lambda>T)ÚkeyÚreverse)r   ra   rb   r   Úindexrd   r*   Úappendr=   ÚgetÚmaxÚscannerÚscore_breakout_candidatere   Úsort)r5   rr   ÚdailyÚsessionsÚucfgÚdÚprevÚcandrk   rl   ÚhistÚpmÚscoredr   ÚphÚpcÚpvÚh20Úresr   rq   r   Úrebuild_listZ   s>   .&
ÿ2((ÿr¥   c              	      s
  t  ˆ¡jtd�}t| ˆ|||ƒ\}}}dd„ |D ƒ}	t| |	ddg |jddd�|jd	d
�ƒ‰i ‰ |D ]K}
|
d }dd„ ˆ |g ¡D ƒ}‡fdd„|| D ƒ}t|ƒdk rVq3||
d d |
d d |
d d t 	|¡dœt 
|¡¥|
d d |
d dœˆ |< q3‡ fdd„|	D ƒ}t tdˆ› d� d¡�}t ˆ||ˆ ddœ|¡ W d   ƒ n1 s«w   Y  ‡fdd„d D ƒ}t td!ˆ› d� d¡�}t ||¡ W d   ƒ n1 sÖw   Y  tˆ› d"�  t ‡ fd#d„|D ƒ¡¡ tjt ˆ¡|d$d%� t|ƒt|ƒt|ƒfS )&NrC   c                 S   s   g | ]}|d  ‘qS )Úsymbolr   ©rU   rŒ   r   r   r   rW   }   s    zbuild_day.<locals>.<listcomp>ÚSPYÚIWMr   rz   r{   r   )r   r¦   c              	   S   óD   g | ]}t |j ¡ ƒt|jƒt|jƒt|jƒt|jƒt|jƒg‘qS r   ©Úintr~   r   r`   r€   r�   r‚   rƒ   rT   r   r   r   rW   ‚   ó
    ,ÿÿc              	      sP   g | ]$}|d  ˆ k rt  |d  ¡jtd�|d |d |d |d |d dœ‘qS )r   rC   r   rt   r&   rx   r   rK   )r   ra   rb   r   rT   rq   r   r   rW   „   s    6ÿé
   rˆ   r‰   rŠ   r   r   )r‰   Úprev_day_highÚrange_20d_highÚ	daily_atrrt   r‹   )r   ÚlevelsÚbaselineÚscorec                    s   g | ]}|ˆ v r|‘qS r   r   ©rU   rk   ©r0   r   r   rW   �   s    Úbars_rA   rY   zT[history] 9:28 list rebuilt with scanner.score_breakout_candidate (build_history.py))r   Útop30_currentÚ	top30_altÚsymbolsÚ_notec                    s$   i | ]}|d d„ ˆ   |g ¡D ƒ“qS )c              	   S   rª   r   r«   )rU   Úxr   r   r   rW   ‘   r­   z(build_day.<locals>.<dictcomp>.<listcomp>)r’   rT   )Úregr   r   Ú
<dictcomp>‘   s
    ÿ

ÿzbuild_day.<locals>.<dictcomp>)r¨   r©   Úbench_ú.jsonc                    sP   g | ]$}|d  ˆ v r|d  |d d |d d |d d |d d dœdœ‘qS )	r¦   rˆ   r‰   rŠ   r   r   rt   )r‰   Úprevious_day_highr°   Úavg_daily_volume)r¦   rˆ   r   r§   r¶   r   r   rW   –   s    þ
þF)Úverbose)r   ra   rb   r   r¥   r=   r’   r*   Ú
volatilityr±   Úfive_day_referencer_   r`   ÚBARSr]   rf   ÚCANDSÚ
write_textÚdumpsÚ	referenceÚbuildr   )r5   rr   r—   r˜   r™   rš   rœ   rŸ   Útopr6   rŒ   rk   rl   r�   Úkeptrm   Úbenchr   )r0   rr   r½   r   Ú	build_dayz   sX   (
ÿ
ÿÿþüÿÿÿ
ÿÿ
þÿrÏ   c                    s   t ƒ r	tdƒ d S tƒ }tƒ d }t ¡ }|jdgt ˆ ¡jt	d�t
dd� t ˆ¡jt	d�t
dd� dd	�d }d
d„ |D ƒ}‡ ‡fdd„|D ƒ}tt|ƒ› d|d › d|d › �dd� t |¡}tdt|ƒ› d�dd� t||ˆ ˆƒ}	t|ƒD ]}}
t ƒ r‡tddd�  nqtd|
› d�  ¡ r¥t|
› d�  ¡ r¥td d |
  ¡ r¥qzt ¡ }z1t||
|	||ƒ\}}}t|
› d|› d|› d|› dt ¡ | d›d t ¡ | d! d›d"�dd� W qz ty÷ } zt|
› d#|› �dd� W Y d }~qzd }~ww td$dd� d S )%Nzmarket hours -- not startingÚuniverser¨   rC   rE   rF   r   rI   )r$   c                 S   s"   g | ]}|d    t¡ ¡  ¡ ‘qS r   rQ   rT   r   r   r   rW   ¦   s   " zmain.<locals>.<listcomp>c                    s(   g | ]}ˆ |  krˆkrn n|‘qS r   r   rµ   ©rg   rh   r   r   rW   §   s   ( z trading days r   z .. ru   Tr'   z	universe z symbolszstopped: 9:15 ETr·   rA   rÀ   r0   rÊ   z: prefilter z	, scored z, kept z  [z.0fz	s, total é<   z min]z	: FAILED ÚHISTORY_DONE)r   r2   r   r   r3   rc   r   ra   rb   r   r   r*   rÐ   Úget_universe_symbolsrn   ÚreversedrÆ   r\   rÇ   ÚTRADErÏ   r1   )rg   rh   r5   r™   Út0Úspyr˜   rG   r6   r—   rr   Út1Ún_cÚn_sÚn_kr<   r   rÑ   r   Úmain�   sD   
"ÿÿ(
&ÿL"€ÿrÝ   Ú__main__rt   )r   ).Ú__doc__r_   r]   Úsysr3   r   r   r   Úpathlibr   Úzoneinfor   Ú__file__ÚresolveÚparentÚHEREÚparentsrÖ   ÚpathÚinsertÚstrÚalpaca.data.requestsr   Úalpaca.data.timeframer	   r
   Úalpaca_clientr   Úconfig_loaderr   rÊ   r”   rÐ   rÄ   r   rÆ   rÇ   rZ   r   r=   rn   r¥   rÏ   rÝ   Ú__name__Úargvr   r   r   r   Ú<module>   s@    

 #ÿ