
    \9jA+                       d Z ddlmZ ddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlmZ ddlmZmZ ddlmZmZmZmZmZ ddlmZmZ d	Z ed
      ZdZdZ dZ! G d de      Z"ddd'dZ#ddddd(dZ$d)dZ%d*dZ&d+dZ'd,dZ(d-dZ)d.dZ*d/dZ+d0dZ,d1dZ-d2d Z.d3d!Z/d4d"Z0d5d#Z1d6d$Z2d7d%Z3e4d&k(  r e5 e3             y)8z;Mirror PyTorch wheel files and generate a pip simple index.    )annotationsN)
HTMLParser)Path)	HTTPErrorURLError)quoteunquoteurljoinurlparse
urlunparse)Requesturlopenz5https://mirrors.tencent.com/pytorch-wheels/whl/cu129/z/data_hdd/wheels/mirror   zpytorch-wheel-mirror/1.0i   c                  (     e Zd Zd fdZddZ xZS )
LinkParserc                0    t         |           g | _        y N)super__init__hrefs)self	__class__s    0/data_hdd/wheels/mirror/mirror_pytorch_wheels.pyr   zLinkParser.__init__   s     "
    c                    |j                         dk7  ry |D ]7  \  }}|j                         dk(  s|s| j                  j                  |       9 y )Nahref)lowerr   append)r   tagattrsnamevalues        r   handle_starttagzLinkParser.handle_starttag"   sH    99;#  	)KD%zz|v%%

!!%(	)r   )returnNone)r    strr!   zlist[tuple[str, str | None]]r%   r&   )__name__
__module____qualname__r   r$   __classcell__)r   s   @r   r   r      s    #)r   r   F	directoryc               .   t        |       }t        j                  |j                        }|j                  j	                  d      s|r|j                  d      dz   }|j                  d      sd|z   }t        |j                  |j                  |dddf      S )N/ )
r   	posixpathnormpathpathendswithrstrip
startswithr   schemenetloc)urlr-   parsedr3   s       r   canonical_urlr;   *   s{    c]Ffkk*D{{C I{{3#%??3Tzv}}fmmT2r2FGGr   GET   )methodheaderstimeoutc               n    dt         i}|r|j                  |       t        | ||      }t        ||      S )Nz
User-Agent)r?   r>   r@   )
USER_AGENTupdater   r   )r9   r>   r?   r@   request_headersrequests         r   request_urlrG   4   s8    #Z0Ow'c?6BG7G,,r   c                  d }t        |dz         D ]\  }	 t        | |      5 }|j                         }|j                  j	                         xs d}|j                  |d      cd d d        c S  t        d|  d	|       # 1 sw Y   nxY w}# t        t        t        t        f$ r3}|}||k  r"t        j                  t        d|z  d             Y d }~d }~ww xY w)
N   rB   utf-8replace)errors      zfailed to fetch : )rangerG   readr?   get_content_charsetdecoder   r   TimeoutErrorOSErrortimesleepminRuntimeError)	r9   r@   retries
last_errorattemptresponsebodycharsetexcs	            r   
fetch_textra   <   s    #'J1% 	/	/S'2 >h}}"**>>@KG{{79{=> >	/ )#b=
>>> > > 8\7; 	/J 

3q'z1-.	/s/   BAB#	BB
	BC')CCc                   t               }|j                  |        g }|j                  D ]  }t        j                  |      j                         }|r|j                  d      r:t        ||      }t        |      }|j                  dvr`|j                  t        |j                  |j                  |j                  dddf              |S )N#>   httphttpsr0   )r   feedr   htmlunescapestripr6   r
   r   r7   r   r   r8   r3   )
index_htmlbase_urlparserlinksr   absoluter:   s          r   extract_linksro   K   s    \F
KK
E Z}}T"((*ts+8T*(#== 11Zv{{BPRTV WXYZ Lr   c                  t        | d      }|g}t               }t               }|r|j                         }t        |d      }||v r%|j                  |       t	        d|        t        |||      }t        ||      D ]  }	t        |	      }
t        |
j                        }|j                  d      r4t        |	d      }|j                  |      r||vr|j                  |       h|j                         j                  d      st        |	      }|j                  |      s|j                  |        |rt        |      S )NTr,   zcrawl r@   rZ   r/   z.whl)r;   setpopaddprintra   ro   r   r	   r3   r4   r6   r   r   sorted)
source_urlr@   rZ   source_rootpendingvisitedwheelscurrentrj   linkr:   decoded_pathdirectory_urlfile_urls                 r   crawl_wheelsr   [   s)   
d;KmGGuF
++-48gGwi !'J
!*g6 	)Dd^F"6;;/L$$S) -dd C ++K8]RY=YNN=1!!#,,V4(.&&{3JJx(	) , &>r   c                    t        |       j                  j                  d      D cg c]  }|st        |       }}d|v r||j	                  d      dz   d  }|rt        | S t        d      S c c}w )Nr/   zpytorch-wheelsrI   .)r   r3   splitr	   indexr   )rw   partpartss      r   source_local_rootr   z   sp    '/
';'@'@'F'Fs'KTttWT]TET5 ekk"23a79: 4</d3i/ Us
   A.A.c                    t        |d      }t        |       t        |      d  }t        |j                  d      D cg c]  }|st	        |       c} }|t        |      z  |z  S c c}w )NTr,   r/   )r;   lenr   r   r	   r   )r9   rw   	dest_rootrx   relative_urlr   relative_paths          r   url_to_local_pathr      sh    
d;K %c+&6&89L\5G5G5LUTPT74=UVM(44}DD Vs   A'A'c                   	 t        | d|      5 }|j                  j                  d      }|t        |      nd cd d d        S # 1 sw Y   y xY w# t        t
        t        t        t        f$ r Y y w xY w)NHEAD)r>   r@   zContent-Length)	rG   r?   getintr   r   rT   rU   
ValueError)r9   r@   r]   sizes       r   remote_sizer      sm    VW= 	;##''(89D $ 03t9d	; 	; 	; xw
C s-   A *A	A A	A A A10A1c                  |j                   j                  dd       |j                  |j                  dz         }t	        | |      }|j                         r.|,|j                         j                  |k(  rd|j                   S |j                         r|d|j                   S d }t        |dz         D ]
  }|j                         r|j                         j                  nd}i }	|rdnd	}
|r	d
| d|	d<   	 t        | |	|      5 }|r|j                  dk7  rd}d	}
|j                  |
dz         5 }	 |j                  t              }|sn|j                  |       *	 d d d        d d d        |j                         j                  }|||k7  rt        d| d|       |j!                  |       d|j                   c S  t        d|  d|       # 1 sw Y   wxY w# 1 sw Y   {xY w# t"        t$        t&        t(        t        f$ r4}|}||k  r"t+        j,                  t/        d|z  d             Y d }~d }~ww xY w)NTparentsexist_okz.partrB   zskip rI   r   abwbzbytes=-Range)r?   r@      r0   zsize mismatch: got z, expected z	download rM   rN   zfailed to download rO   )parentmkdir	with_namer"   r   existsstatst_sizerP   rG   statusopenrQ   
CHUNK_SIZEwriterY   rK   r   r   rT   rU   rV   rW   rX   )r9   
local_pathr@   rZ   	part_pathexpected_sizer[   r\   resume_fromr?   moder]   outputchunkdownloaded_sizer`   s                   r   download_filer      sH   D48$$Z__w%>?IW5M}8Z__=N=V=VZg=gz'((}4z'((#'J1% /2;2B2B2Dinn&..!"$"t!'}A6GG	/S'7C 	,x8??c#9"#KD^^D2I. ,& (j 9$!U+	  "	,		, (nn.66O(_-M"%88IUbTc#deej)z/001/< ,SEJ<@
AA%, ,		, 	," 8\7LI 	/J 

3q'z1-.	/sI   <G/
*G#4,G!G#)AG/G G##G,	(G//I)H<<Ic                L    t        j                  dd|       j                         S )Nz[-_.]+r   )resubr   )r"   s    r   normalize_package_namer      s    66)S$'--//r   c                >    t        | j                  dd      d         S )Nr   rI   r   )r   r   )filenames    r   wheel_package_namer      s    !(..a"8";<<r   c                   t        j                         }| j                  d      5 t        fdd      D ]  }|j	                  |        	 d d d        |j                         S # 1 sw Y   |j                         S xY w)Nrbc                 .     j                  t              S r   )rQ   r   )
wheel_files   r   <lambda>zsha256_file.<locals>.<lambda>   s    *//*"= r   r   )hashlibsha256r   iterrD   	hexdigest)r3   digestr   r   s      @r   sha256_filer      sq    ^^F	4 !J=sC 	!EMM% 	!! ! s   &A&&A>c                d    | j                   j                  dd       | j                  |d       y )NTr   rJ   )encoding)r   r   
write_text)r3   r^   s     r   
write_htmlr      s)    KKdT2OOD7O+r   c          
     x   t        |j                  d            }i }|D ]6  }|j                  t        |j                        g       j                  |       8 | dz  }g }t        |      D ]  }|j                  dt        |       dt        j                  |       d       g }	||z  }
||   D ]  }t        j                  j                  ||
      j                  t        j                  d      }t        |d      }|r|d	t        |       z  }|	j                  d| d
t        j                  |j                         d        t        |
dz  t!        dj#                  |	                    t        |dz  t!        dj#                  |                   t%        dt'        |       dt'        |       d       y )Nz*.whlsimplez	<a href="z/">z</a>r/   z/._-+~)safez#sha256=z">z
index.html
zindexed z wheels in z	 packages)rv   rglob
setdefaultr   r"   r   r   rg   escapeosr3   relpathrK   sepr   r   	html_pagejoinru   r   )r   
wheel_roothashesr{   packageswheelsimple_rootproject_linkspackagewheel_linkspackage_dirrelativer   s                r   generate_simple_indexr      s   J$$W-.F&(H N.uzz:B?FFuMN h&KM(# Ryw(8DKK<P;QQUVW!G+g& 	REwwuk:BB2663OH1D(;u#5"6774&4;;uzz3J2K4PQ	R 	;-y;9O/PQR {\)9TYY}5M+NO	HS[MS]O9
EFr   c                    d|  dS )Nz<!doctype html>
<html><body>
z
</body></html>
 )rm   s    r   r   r      s    ,UG3EFFr   c                    t        j                  d      } | j                  dt        dt                | j                  dt        t
        dt
                | j                  d	t        t        d
t                | j                  dt        dd       | j                  dt        dd       | j                  ddd       | j                  ddd       | j                         S )Nz6Mirror PyTorch wheels and generate a pip simple index.)descriptionz--sourcezsource URL, default: )defaulthelpz--destzmirror root, default: )typer   r   z	--workerszdownload workers, default: z	--timeoutr=   zHTTP timeout in secondsz	--retries   znetwork retry countz	--dry-run
store_truez)crawl and list wheels without downloading)actionr   z--no-hashesz1do not add sha256 fragments to simple index links)	argparseArgumentParseradd_argumentDEFAULT_SOURCEr   DEFAULT_DESTr   DEFAULT_WORKERS
parse_args)rl   s    r   r   r      s    $$1ijF

NCXYgXhAij
t\J`am`nHop
#Ojkzj{M|}
#r@YZ
#q?TU
L?jk
lAtur   c                 .   t               } | j                  dk  rt        dt        j                         yt        | j                  d      }| j                  j                         }|t        |      z  }t        || j                  | j                        }t        dt        |       d	       | j                  r|D ]  }t        |        y
d
}t        j                   j#                  | j                        5 }|D ci c]<  }|j%                  t&        |t)        |||      | j                  | j                        |> }}t        j                   j+                  |      D ]!  }	||	   }	 t        |	j-                                # 	 d d d        t1        ||| j2                          |r t        d| dt        j                         yy
c c}w # t.        $ r0}
|dz  }t        d| d|
 t        j                         Y d }
~
d }
~
ww xY w# 1 sw Y   xY w)NrI   z--workers must be at least 1)filerM   Tr,   rq   zfound z wheelsr   )max_workerszerror rO   )r   zcompleted with z download failures)r   workersru   sysstderrr;   sourcedestresolver   r   r@   rZ   r   dry_run
concurrentfuturesThreadPoolExecutorsubmitr   r   as_completedresult	Exceptionr   	no_hashes)argsr   r   r   r{   r9   failuresexecutorfuture_to_urlfuturer`   s              r   mainr     s   <D||a,3::>4;;$7F		!!#I.v66J&$,,MF	F3v;-w
'(|| 	C#J	H				.	.4<<	.	H >H 
 OOM30A#vy0Ycgcocoy}  zF  zFO  G  IL  L
 
 !((55mD 	>F'C>fmmo&	>> )ZDNN8JKz);<3::N!
  >Ase2cU+#**==>> >sI   :H?AG
 )H*GH
H	H&H>HHHH__main__)r9   r'   r-   boolr%   r'   )r9   r'   r>   r'   r?   zdict[str, str] | Noner@   r   )r9   r'   r@   r   rZ   r   r%   r'   )rj   r'   rk   r'   r%   	list[str])rw   r'   r@   r   rZ   r   r%   r
  )rw   r'   r%   r   )r9   r'   rw   r'   r   r   r%   r   )r9   r'   r@   r   r%   z
int | None)
r9   r'   r   r   r@   r   rZ   r   r%   r'   )r"   r'   r%   r'   )r   r'   r%   r'   )r3   r   r%   r'   )r3   r   r^   r'   r%   r&   )r   r   r   r   r   r	  r%   r&   )rm   r'   r%   r'   )r%   zargparse.Namespace)r%   r   )6__doc__
__future__r   r   concurrent.futuresr   r   rg   r   r1   r   r   rV   html.parserr   pathlibr   urllib.errorr   r   urllib.parser   r	   r
   r   r   urllib.requestr   r   r   r   r   rC   r   r   r;   rG   ra   ro   r   r   r   r   r   r   r   r   r   r   r   r   r  r(   
SystemExitr   r   r   <module>r     s    A "     	  	 
  "  , F F + I-.'


) 
) 27 H ,1SWhj -? >0E)BX0=,
G4G	$N z
TV
 r   