o
    $Íj"  ã                   @   s
  d Z ddlZddlZddlZddlZddlmZ ddlmZmZ ddl	m
Z
 ddlmZ ddlZe
eƒ ¡ jZe
dƒZed d	 Zej deeƒ¡ ed
ƒZd\ZZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Ze dkrƒdej!v r~eej!ej! "d¡d  ƒ dS eƒ  dS dS )a@  
news_test.py -- [2026-10-12] The user: does NEWS (Benzinga via Alpaca, trade/news_recorder.py tags) predict which scanner
picks run, and does it help or hurt the bot's buys? 2 years (Oct 2024 - Oct 2026).
  backfill   python3 news_test.py --backfill 2024-10-01..2026-10-09   (data/news/<day>.jsonl for every trading day)
  test       python3 news_test.py
PICKS  = the daily top-30 lists (reports/open_window/bars). News known AT THE OPEN = from the previous trading day 16:00
         to 9:30. Outcome of the stock-day: reached +3% / +5% over the open, close vs open.
TRADES = the 'BASE' trades of avoid_rules_test.jsonl (live v2 buys + live exits, every signal), at the real 0.2% cost.
         News known AT THE BUY = previous day 16:00 until the buy minute.
Per tag and "any news" vs "no news", year 1 (< 2025-10-01) vs year 2.
é    N)Údefaultdict)ÚdatetimeÚ	timedelta)ÚPath)ÚZoneInfoz/var/www/screener/tradeÚreportsÚopen_windowzAmerica/New_York)gš™™™™™É?z
2025-10-01c                   C   s   t dd„ t d¡D ƒƒS )Nc                 s   s   � | ]
}|j d d… V  qdS )é   é   N)Úname)Ú.0Úp© r   únews_test.pyÚ	<genexpr>    s   € zdays_list.<locals>.<genexpr>zbars/bars_20*.json.gz)ÚsortedÚOWÚglobr   r   r   r   Ú	days_list   s   r   c                    s.  dd l }ddlm} |ƒ }|j|jdœ}|  d¡\‰ ‰‡ ‡fdd„tƒ D ƒD ]j}td d |› d	� }| ¡ r?| 	¡ j
d
kr?q&t |¡jtd�}tdƒD ]6}z| |||tdd� | |¡¡}	W  n! ty‚ }
 zt|d|
dd� t d¡ d}	W Y d }
~
qLd }
~
ww t||	ddd� t d¡ q&tdƒ d S )Nr   )Úget_env)zAPCA-API-KEY-IDzAPCA-API-SECRET-KEYz..c                    s(   g | ]}ˆ |  krˆkrn n|‘qS r   r   )r   Úd©ÚaÚbr   r   Ú
<listcomp>(   s   ( zbackfill.<locals>.<listcomp>ÚdataÚnewsz.jsonliè  ©Útzinfoé   é   ©ÚdaysÚretryT)Úflushé
   Úarticlesg      à?ÚNEWS_BACKFILL_DONE)Únews_recorderÚconfig_loaderr   Úapi_keyÚ
secret_keyÚsplitr   ÚTRADEÚexistsÚstatÚst_sizer   ÚfromisoformatÚreplaceÚETÚrangeÚpullr   Ú	load_seenÚ	ExceptionÚprintÚtimeÚsleep)ÚrngÚNRr   ÚeÚhÚdayr   Úd0ÚattemptÚnÚexr   r   r   Úbackfill#   s&   &*€ÿrD   c                  C   sž   t tƒ} ttd d  d¡ƒD ].}t|ƒD ]'}t |¡}t 	|d ¡j
td�}|d D ]}| |  ||d p8dgf¡ q+qq| D ]}| | jd	d
„ d� q@| S )z#sym -> sorted [(datetime ET, tags)]r   r   z	20*.jsonlr9   r   ÚsymbolsÚtagsÚuntaggedc                 S   ó   | d S )Nr   r   )Úxr   r   r   Ú<lambda>A   ó    znews_index.<locals>.<lambda>)Úkey)r   Úlistr   r-   r   ÚopenÚjsonÚloadsr   r1   r2   r3   ÚappendÚsort)Úidxr   ÚlÚrÚtÚsr   r   r   Ú
news_index7   s   
ÿýrX   c                 C   sT   t ƒ }d}|  |d¡D ]\}}||  kr|k r%n q|d7 }|t |ƒO }q||fS )Nr   r   r    )ÚsetÚget)rS   ÚsymÚstartÚendrF   rB   rV   Útgr   r   r   ÚwindowE   s   
€r_   c                     sh  t ƒ } tƒ ‰ ‡ fdd„tˆ ƒD ƒ}tdtdd„ |  ¡ D ƒƒ› dt| ƒ› d�ƒ g }ˆ D ]‰}||vr2q+t t	 
td d	|› d
� d¡¡}t || ¡jdtd�}t |¡jddtd�}|d D ]T}|d  |¡phi  d¡pmg }t|ƒdk ruq^|d d tdd„ |D ƒƒ|d d }	}
}t| |||ƒ\}}| ||tk|||
|	d k|
|	d k||	 d d dœ¡ q^~q+dd„ }tdt|ƒ› d �ƒ t|d!d"d#„ |D ƒƒƒ t|d$d%d#„ |D ƒƒƒ td&d'„ |D ƒƒ}|D ]‰‡fd(d#„|D ƒ}t|ƒd)krþt|ˆ|ƒƒ qætd* }| ¡ �std+ƒ d S d,d#„ t
|ƒD ƒ}d-d#„ |D ƒ}|D ]@‰t | ˆd. ˆd. ¡¡jdtd�}t ˆd. ¡jddtd�tˆd/ d0� }t| ˆd1 ||ƒ\ˆd2< ˆd3< ˆd4 t ˆd5< �q d6d7„ }td8t|ƒ› d9t› d:�ƒ t|d!d;d#„ |D ƒƒƒ t|d$d<d#„ |D ƒƒƒ td=d'„ |D ƒƒD ]‰‡fd>d#„|D ƒ}t|ƒd?k�r¬t|ˆ|ƒƒ �q“td@ƒ d S )ANc                    s"   i | ]\}}|r|ˆ |d   “qS )r    r   )r   Úir   r!   r   r   Ú
<dictcomp>P   s   " zmain.<locals>.<dictcomp>znews index: c                 s   s   � | ]}t |ƒV  qd S )N)Úlen)r   Úvr   r   r   r   Q   ó   € zmain.<locals>.<genexpr>z symbol-articles, z symbolsÚbarsÚbars_z.json.gzÚrté   )Úhourr   é	   é   )ri   Úminuter   Útop30_currentrE   é<   r   r    c                 s   s   � | ]}|d  V  qdS )é   Nr   )r   rI   r   r   r   r   ]   rd   éÿÿÿÿé   g{®Gázð?gÍÌÌÌÌÌð?éd   )r?   Úy2rB   rF   Úup3Úup5Úclosec                    sÞ   |s	d| d›d�S g }ddd„ fddd„ ffD ]K\}‰ ‡ fd	d
„|D ƒ}|  |› dt|ƒd›d|r;t dd
„ |D ƒ¡ndd›d|rKt dd
„ |D ƒ¡ndd›d|r[t dd
„ |D ƒ¡ndd›d�
¡ qd| d›d�d |¡ S )Nú  Ú28z 0Úy1c                 S   s
   | d  S ©Nrs   r   ©rU   r   r   r   rJ   e   s   
 z)main.<locals>.pick_line.<locals>.<lambda>rs   c                 S   rH   rz   r   r{   r   r   r   rJ   e   rK   c                    s   g | ]}ˆ |ƒr|‘qS r   r   ©r   rU   ©Úselr   r   r   f   ó    z+main.<locals>.pick_line.<locals>.<listcomp>ú Ú5z | +3% c                 S   ó   g | ]}|d  ‘qS )rt   r   r|   r   r   r   r   g   ó    r   z4.0%z +5% c                 S   r‚   )ru   r   r|   r   r   r   r   g   rƒ   z close c                 S   r‚   )rv   r   r|   r   r   r   r   h   rƒ   z+.2fú%ú || )rQ   rb   ÚnpÚmeanÚjoin©ÚlabÚXÚoutÚnmÚZr   r}   r   Ú	pick_linea   s    Vÿzmain.<locals>.pick_linez
PICKS (top-30 stock-days: z.): news known at the open (prev 16:00 -> 9:30)zno newsc                 S   ó   g | ]}|d  s|‘qS ©rB   r   r|   r   r   r   r   k   r   zmain.<locals>.<listcomp>zANY newsc                 S   ó   g | ]}|d  r|‘qS r‘   r   r|   r   r   r   r   l   r   c                 S   ó   h | ]}|d  D ]}|’qqS ©rF   r   )r   rU   rV   r   r   r   Ú	<setcomp>m   ó    zmain.<locals>.<setcomp>c                    ó   g | ]
}ˆ |d  v r|‘qS r”   r   r|   )rV   r   r   r   o   ó    é(   zavoid_rules_test.jsonlz6
(no avoid_rules_test.jsonl yet -- trade part skipped)c                 S   s   g | ]}t  |¡‘qS r   )rO   rP   )r   rT   r   r   r   r   v   s    c                 S   s   g | ]
}|d  dkr|‘qS )ÚrunÚBASEr   ©r   rV   r   r   r   r   w   r˜   r?   Úbuy_m)Úminutesr[   rB   rF   Úpl_grossÚplc                    s¦   g }ddd„ fddd„ ffD ]8\}‰ t  ‡ fdd„|D ƒ¡}| |› dt|ƒd	›dt|ƒr1| ¡ nd
d›dt|ƒr@t  |d
k¡nd
d›�¡ qd| d›d�d |¡ S )Nry   c                 S   s   | d t k S ©Nr?   ©ÚSPLITr{   r   r   r   rJ      ó    z*main.<locals>.trade_line.<locals>.<lambda>rs   c                 S   s   | d t kS r¡   r¢   r{   r   r   r   rJ      r¤   c                    s   g | ]
}ˆ |ƒr|d  ‘qS )r    r   r|   r}   r   r   r   €   r˜   z,main.<locals>.trade_line.<locals>.<listcomp>r€   r�   r   z+.3fz%/trade win z.0%rw   rx   r…   )r†   ÚarrayrQ   rb   r‡   rˆ   r‰   r   r}   r   Ú
trade_line}   s
    Rzmain.<locals>.trade_linez%
TRADES (live v2 buys, every signal: z5): news known at the buy (prev 16:00 -> buy minute), z% costc                 S   r�   r‘   r   rœ   r   r   r   r   „   r   c                 S   r’   r‘   r   rœ   r   r   r   r   …   r   c                 S   r“   r”   r   )r   rV   rI   r   r   r   r•   †   r–   c                    r—   r”   r   rœ   )r^   r   r   r   ‡   r˜   é   ÚNEWS_TEST_DONE)rX   r   Ú	enumerater8   ÚsumÚvaluesrb   rO   ÚloadÚgziprN   r   r   r1   r2   r3   rZ   Úmaxr_   rQ   r£   r   ÚHEREr.   r   ÚCOST)rS   ÚprevÚPr   ÚddÚstÚoprW   r   ÚOÚHÚCrB   rF   r�   Úalltagsr‹   ÚtfÚTÚbtr¦   r   )r"   rV   r^   r   ÚmainM   s`   ,"..<	€
$(€r½   Ú__main__z
--backfillr    )#Ú__doc__r­   rO   Úsysr9   Úcollectionsr   r   r   Úpathlibr   Úzoneinfor   Únumpyr†   Ú__file__ÚresolveÚparentr¯   r-   r   ÚpathÚinsertÚstrr3   r°   r£   r   rD   rX   r_   r½   Ú__name__ÚargvÚindexr   r   r   r   Ú<module>   s4    @

ü