o
    2BÑj  ã                   @   sæ   d Z ddlZddlZddlZddlZddlmZ ddlmZmZm	Z	 ddl
mZ ddlmZ ej deeeƒ ¡ jƒ¡ ddlmZmZ edƒZeeƒ ¡ jd	 Zd
d„ Zdd„ Zdd„ Zddd„Zedkrqeejdd… Ž  dS dS )a?  
massive_bars_10y.py -- [2026-10-05] 10 YEARS of 1-minute bars (Massive, ex-Polygon, Developer plan) for every
US stock that opened at $5-15 with >= 500k shares that day (~500/day), to choose playbook references from
later (the user: download 10 years, select later). Trades/flow are NOT downloaded here (too big for all):
only for the selected stock-days, later.
  per day: grouped daily bars (1 call) -> list; then 1 minute-aggregate call per stock (8 in parallel)
  output: reports/open_window/bars10y/<YYYY>/<date>.json.gz  {"date", "daily": {sym: [o,h,l,c,v,prev_close]},
          "bars": {sym: [[epoch_s, o, h, l, c, v], ...] 9:30-16:00}}
Resumable (skips finished days), newest day first. Never runs 9:10-16:05 on weekdays (market hours).
    python3 massive_bars_10y.py [START] [END]      default 2016-10-03 .. 2026-10-02
é    N)ÚThreadPoolExecutor)ÚdateÚdatetimeÚ	timedelta)ÚPath)ÚZoneInfo©ÚgetÚBASEzAmerica/New_YorkÚbars10yc                  C   s6   t  t¡} |  ¡ dk od| j| jf  kodk S   S )Né   )é	   é
   )é   r   )r   ÚnowÚETÚweekdayÚhourÚminute)Út© r   úmassive_bars_10y.pyÚmarket_hours   s   
,r   c                 C   s.   t t› d| › �ddiƒ}dd„ |  dg ¡D ƒS )Nz)/v2/aggs/grouped/locale/us/market/stocks/ÚadjustedÚfalsec                    s,   i | ]‰ t ‡ fd d„dD ƒƒrˆ d ˆ “qS )c                 3   s   � | ]}|ˆ v V  qd S ©Nr   )Ú.0Úk©Úrr   r   Ú	<genexpr>#   s   € z%grouped.<locals>.<dictcomp>.<genexpr>©ÚoÚhÚlÚcÚvÚT)Úall)r   r   r   r   Ú
<dictcomp>#   s   , zgrouped.<locals>.<dictcomp>Úresultsr   )ÚdÚjr   r   r   Úgrouped!   s   r-   c                    s~   t t› d| › d|› d|› �ddddœƒ}t t|ƒ¡jtd�}|jd	d
d� ¡ |jdd� ¡ ‰ ‰‡ ‡fdd„|  dg ¡D ƒS )Nz/v2/aggs/ticker/z/range/1/minute/ú/r   ÚasciPÃ  )r   ÚsortÚlimit)Útzinfor   é   )r   r   r   )r   c              	      s\   g | ]*}ˆ |d  d   krˆk rn nt |d  d ƒ|d |d |d |d |d g‘qS )r   iè  r"   r#   r$   r%   r&   )Úint)r   Úb©ÚaÚzr   r   Ú
<listcomp>*   s     .ÿzminute_bars.<locals>.<listcomp>r*   )r	   r
   r   ÚfromisoformatÚstrÚreplacer   Ú	timestamp)Úsymr+   r,   Úd0r   r6   r   Úminute_bars&   s   ($r@   ú
2016-10-03ú
2026-10-02c                    sœ  g t  |¡}‰ ˆ t  | ¡kr(ˆ  ¡ dk r| ˆ ¡ ˆ tdd�8 ‰ ˆ t  | ¡ksd }t ¡ d}}|D �]‰ ttˆ jƒ ˆ › d� }| 	¡ rGq3t
ƒ rStddd�  d S ztˆ ƒ‰W n* tyƒ } zd	t|ƒv r~tˆ › d
�dd� tddd� W Y d }~ d S ‚ d }~ww ˆs‡q3ˆ tdd� i }‰tdƒD ] }	| ¡ dk r®t|ƒ}
|
r®dd„ |
 ¡ D ƒ‰ n|tdd�8 }q•dd„ ˆ ¡ D ƒ}tdƒ�}tt|| ‡ fdd„|¡ƒƒ}W d   ƒ n1 sÝw   Y  |jjddd� | d¡}t |d¡�#}t tˆ ƒ‡‡fdd„|D ƒdd„ | ¡ D ƒdœ|¡ W d   ƒ n	1 �sw   Y  | |¡ |d7 }tˆ › dt|ƒ› dt ¡ | d d›d|› d�dd� q3tddd� d S ) Nr   é   )Údaysr   z.json.gzzstopped: market hoursT)ÚflushÚNOT_AUTHORIZEDzI: older than the plan's 10-year history limit -- nothing more to downloadÚBARS10Y_DONEé   c                 S   s   i | ]	\}}||d  “qS )r%   r   )r   r   r&   r   r   r   r)   L   ó    zmain.<locals>.<dictcomp>c                 S   sD   g | ]\}}d |d   krdkrn n|d dkr|  ¡ r|‘qS )r   r"   é   r&   i ¡ )Úisalpha)r   Úsr   r   r   r   r9   O   s   D zmain.<locals>.<listcomp>é   c                    s
   t | ˆ ƒS r   )r@   )rL   )r+   r   r   Ú<lambda>Q   s   
 zmain.<locals>.<lambda>)ÚparentsÚexist_okz.tmpÚwtc                    sL   i | ]"}|ˆ| d  ˆ| d ˆ| d ˆ| d ˆ| d ˆ   |¡g“qS r!   )r	   )r   rL   )ÚpcÚtodayr   r   r)   V   s   L c                 S   s   i | ]	\}}|r||“qS r   r   )r   rL   r5   r   r   r   r)   W   rI   )r   ÚdailyÚbarsz: z	 stocks [é<   z.0fz min, z days])r   r:   r   Úappendr   ÚtimeÚOUTr;   ÚyearÚexistsr   Úprintr-   ÚRuntimeErrorÚrangeÚitemsr   ÚdictÚzipÚmapÚparentÚmkdirÚwith_suffixÚgzipÚopenÚjsonÚdumpÚrenameÚlen)ÚstartÚendrD   ÚprevÚt0ÚnÚfÚeÚpd_Ú_ÚgÚsymsÚexrU   ÚtmpÚfhr   )r+   rR   rS   r   Úmain.   sf   
ý
€û
 ÿ

þþÿ
8rz   Ú__main__rC   é   )rA   rB   )Ú__doc__rf   rh   ÚsysrX   Úconcurrent.futuresr   r   r   r   Úpathlibr   Úzoneinfor   ÚpathÚinsertr;   Ú__file__Úresolverc   Úmassive_trialr	   r
   r   rY   r   r-   r@   rz   Ú__name__Úargvr   r   r   r   Ú<module>   s(    
0ÿ