o
    x Íjb  ã                   @   sê   d Z ddlZddlZddlZddlZddlmZmZ ddlmZ ddl	m
Z
 ddlZeeƒ ¡ jZej deeƒ¡ ddlmZ e
dƒZdZg d	¢Zd
d„ eD ƒZdd„ Zddd„Zdd„ Zdd„ Zdd„ Zdd„ Zedkrseƒ  dS dS )an  
news_recorder.py -- [2026-10-12] RECORD ONLY (the user): every Benzinga headline (Alpaca news API, v1beta1/news) for ALL
stocks, polled every 60 s from 6:00 to 20:00 ET on weekdays, so we can later test whether news predicts which scanner
picks run. Nothing live reads it.
Output: data/news/<date>.jsonl, one line per article (first time seen): id, time (ET), headline, summary, symbols,
source, author, url, tags. Tags are simple keyword groups (a starting point, to be tested):
  analyst_up / analyst_down / analyst_init / analyst_other, price_target_up / price_target_down, earnings, guidance,
  why_moving, halt, offering, fda, merger, contract, insider, legal, management, buyback, dividend
Logs: logs/news_recorder.log. cron 6:00 weekdays (flock); it exits at 20:00.
    python3 news_recorder.py            python3 news_recorder.py --backfill 2026-10-12   (one past day)
é    N)ÚdatetimeÚ	timedelta)ÚPath)ÚZoneInfo)Úget_envzAmerica/New_Yorkz(https://data.alpaca.markets/v1beta1/news))Ú
analyst_upz;\bupgrade[sd]?\b|\braises? .*to (buy|outperform|overweight))Úanalyst_downz\bdowngrade[sd]?\b)Úanalyst_initz?\binitiates? coverage\b|\bbegins coverage\b|\bstarts coverage\b)Úanalyst_otherz\bmaintains\b|\breiterates?\b)Úprice_target_upzraises price target)Úprice_target_downzlowers price target)ÚearningszL\bearnings\b|\bq[1-4] (20\d\d )?results\b|\beps\b|\brevenue\b|\btranscript\b)Úguidancez%\bguidance\b|\boutlook\b|\bforecast\b)Ú
why_movingz`what'?s going on|why is .* (stock|shares)|shares are trading (higher|lower)|stocks moving|movers)Úhaltz\bhalted\b|\bcircuit breaker\b)Úofferingz@\boffering\b|\bprivate placement\b|\bregistered direct\b|\bdilut)Úfdaz0\bfda\b|\bphase [123i]+\b|\bclinical\b|\btrial\b)Úmergerz>\bacquir|\bmerger\b|\bto be acquired\b|\bbuyout\b|\btakeover\b)ÚcontractzQ\bcontract\b|\bpartnership\b|\bpartners with\b|\bagreement\b|\bawarded\b|\bdeal\b)Úinsiderz)\binsider\b|\bform 4\b|\bbuys .* shares\b)Úlegalzb\blawsuit\b|\bcourt\b|\bsec (charges|probe)|\binvestigation\b|\bsettlement\b|\brestraining order\b)Ú
managementz9\bceo\b|\bcfo\b|\bappoints?\b|\bresigns?\b|\bsteps down\b)Úbuybackz\bbuyback\b|\brepurchase\b)Údividendz\bdividend\bc                 C   s"   g | ]\}}|t  |t j¡f‘qS © )ÚreÚcompileÚI)Ú.0ÚkÚpr   r   únews_recorder.pyÚ
<listcomp>1   ó   " r"   c                    s   ‡ fdd„t D ƒS )Nc                    s"   g | ]\}}|  ˆ pd ¡r|‘qS )Ú )Úsearch)r   r   Úrx©Útextr   r!   r"   5   r#   ztag.<locals>.<listcomp>)ÚTAG_REr'   r   r'   r!   Útag4   s   r*   c                 C   s\   |  ¡ |  ¡ ddddœ}|r||d< tjt| |dd�}| ¡  | ¡ }| dg ¡| d	¡fS )
Né2   ÚascÚfalse)ÚstartÚendÚlimitÚsortÚinclude_contentÚ
page_tokené   )ÚheadersÚparamsÚtimeoutÚnewsÚnext_page_token)Ú	isoformatÚrequestsÚgetÚURLÚraise_for_statusÚjson)Úhr.   r/   Útokenr    ÚrÚjr   r   r!   Úfetch8   s   rD   c                 C   s4  d}| D ]“}|d |v rq|  |d ¡ t |d  dd¡¡ t¡}|d | d¡| d¡| d¡p3d	d d
… | d¡p=g | d¡| d¡| d¡t| d¡pPd	d | d¡pXd	 ƒdœ	}t	d d | 
¡  ¡ › d� }|jjddd� t|dƒ�}| t |¡d ¡ W d   ƒ n1 sŽw   Y  |d7 }q|S )Nr   ÚidÚ
created_atÚZz+00:00z%Y-%m-%d %H:%M:%SÚheadlineÚsummaryr$   iô  ÚsymbolsÚsourceÚauthorÚurlú )	rE   ÚtimerH   rI   rJ   rK   rL   rM   ÚtagsÚdatar8   ú.jsonlT)ÚparentsÚexist_okÚaÚ
é   )Úaddr   ÚfromisoformatÚreplaceÚ
astimezoneÚETÚstrftimer<   r*   ÚBASEÚdater:   ÚparentÚmkdirÚopenÚwriter?   Údumps)ÚartsÚseenÚnrU   ÚtÚrowr    Úfhr   r   r!   ÚsaveB   s"   (2þÿ
rk   c                 C   s6   t d d | › d� }| ¡ rdd„ t|ƒD ƒS tƒ S )NrQ   r8   rR   c                 S   s   h | ]	}t  |¡d  ’qS )rE   )r?   Úloads)r   Úlr   r   r!   Ú	<setcomp>V   s    zload_seen.<locals>.<setcomp>)r^   Úexistsrb   Úset)Údayr    r   r   r!   Ú	load_seenT   s    rr   c                 C   s4   d\}}	 t | |||ƒ\}}|t||ƒ7 }|s|S q)N)r   N)rD   rk   )r@   r.   r/   rf   rg   rA   re   r   r   r!   ÚpullY   s   ürs   c            	   
   C   s†  t ƒ } | j| jdœ}dtjv r;tjtj d¡d  }t |¡jt	d�}t
|ƒ}t|t|||tdd� |ƒddd� d S t t	¡}t
| ¡  ¡ ƒ}|jd	d	d	d	d
�tdd� }td|d›d�dd� 	 t t	¡}|jdkrmnFz t||tdd� ||ƒ}|rŠtd|d›d|› d�dd� |}W n ty¬ } ztd|d›d|› �dd� W Y d }~nd }~ww t d¡ qbtdt t	¡d›d�dd� d S )N)zAPCA-API-KEY-IDzAPCA-API-SECRET-KEYz
--backfillrW   )Útzinfo)Údayszarticles savedT)Úflushr   )ÚhourÚminuteÚsecondÚmicrosecondé   )Úhoursú[z%H:%M:%Sz] news recorder startr4   é
   )Úminutesz] +z	 articlesz] error é<   z] stop)r   Úapi_keyÚ
secret_keyÚsysÚargvÚindexr   rY   rZ   r\   rr   Úprintrs   r   Únowr_   r:   rw   Ú	ExceptionrO   Úsleep)	Úer@   rq   Úd0rf   r‡   Úsincerg   Úexr   r   r!   Úmainb   s6   
$


&€ÿ
õ rŽ   Ú__main__)N) Ú__doc__r?   r   rƒ   rO   r   r   Úpathlibr   Úzoneinfor   r;   Ú__file__Úresolver`   r^   ÚpathÚinsertÚstrÚconfig_loaderr   r\   r=   ÚTAGSr)   r*   rD   rk   rr   rs   rŽ   Ú__name__r   r   r   r!   Ú<module>   s2    

	
ÿ