o j@@sldZddlmZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl mZmZmZmZddlmZddlmZddlmZddlmZmZdd lmZdd lmZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9m:Z:m;Z;e ee?e?fZ@dWddZAGdddeBZCdXd"d#ZDGd$d%d%eBZEdYd(d)ZF *dZd[d-d.ZGd\d1d2ZHGd3d4d4ZIGd5d6d6eZJd]d8d9ZKeKd^d=d>ZLed?d@GdAd;d;ZMGdBdCdCeZN d_d`dJdKZO ?dadbdMdNZPd*dOdcdQdRZQGdSdTdTeZRGdUdVdVZSdS)dzO The main purpose of this module is to expose LinkCollector.collect_sources(). ) annotationsN)CallableIterableMutableMappingSequence) dataclass) HTMLParser)Values) NamedTupleProtocol)canonicalize_name)Response) RetryError)ConnectionFailedErrorConnectionTimeoutErrorNetworkConnectionErrorProxyConnectionErrorSSLMissingErrorSSLVerificationError)Link) SearchScope) PipSession)raise_for_status)is_archive_fileredact_auth_from_url) url_to_path)vcs)CandidatesFromPage LinkSource build_sourceurlstrreturn str | NonecCs6tjD]}||r|t|dvr|SqdS)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rschemeslower startswithlen)r"schemer+Z/var/www/kodo/Anonymous/send/lib/python3.10/site-packages/pip/_internal/index/collector.py_match_vcs_scheme4s r-cseZdZdfdd ZZS) _NotAPIContent content_typer# request_descr$Nonecst||||_||_dSN)super__init__r/r0)selfr/r0 __class__r+r,r4@s z_NotAPIContent.__init__)r/r#r0r#r$r1)__name__ __module__ __qualname__r4 __classcell__r+r+r6r,r.?sr.responser r1cCs2|jdd}|}|drdSt||jj)z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr'r(r.requestmethod)r<r/content_type_lr+r+r,_ensure_api_headerFsrEc@s eZdZdS)_NotHTTPN)r8r9r:r+r+r+r,rF\srFsessionrcCsFtj|\}}}}}|dvrt|j|dd}t|t|dS)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httphttpsT)allow_redirectsN)urllibparseurlsplitrFheadrrE)r"rGr*netlocpathqueryfragmentrespr+r+r,_ensure_api_response`s  rTFforce_revalidateboolcCstt|jr t||dtdt|ddgdi}|r)tdd|d<|j||d }t |t |td t||j d d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. )rGzGetting page %sAcceptz, )r?z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z"Refreshing package index response.z max-age=0z Cache-Control)r@zFetched page %s as %sr=r>) rrfilenamerTloggerdebugrjoinrArrEr@)r"rGrUr@rSr+r+r,_get_simple_responsers&   r\r@ResponseHeaderscCs<|rd|vrtj}|d|d<|d}|rt|SdS)z=Determine if we have any encoding information in our headers.r=z content-typecharsetN)emailmessageMessage get_paramr#)r@mr^r+r+r,_get_encoding_from_headerss    rdc@s*eZdZdddZdd d Zdd dZdS)CacheablePageContentpage IndexContentr$r1cCs|jsJ||_dSr2)cache_link_parsingrfr5rfr+r+r,r4s  zCacheablePageContent.__init__otherobjectrVcCst|t|o|jj|jjkSr2) isinstancetyperfr")r5rjr+r+r,__eq__szCacheablePageContent.__eq__intcCs t|jjSr2)hashrfr"r5r+r+r,__hash__s zCacheablePageContent.__hash__N)rfrgr$r1)rjrkr$rV)r$ro)r8r9r:r4rnrrr+r+r+r,res  rec@seZdZdddZdS) ParseLinksrfrgr$Iterable[Link]cCsdSr2r+rir+r+r,__call__szParseLinks.__call__Nrfrgr$rt)r8r9r:rur+r+r+r,rssrsfncs2tjd fdd td fd d }|S)z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. cacheable_pagerer$ list[Link]cst|jSr2)listrf)rx)rwr+r,wrappersz*with_cached_index_content..wrapperrfrgcs|jr t|St|Sr2)rhrerz)rfrwr{r+r,wrapper_wrappers  z2with_cached_index_content..wrapper_wrapperN)rxrer$ry)rfrgr$ry) functoolscachewraps)rwr}r+r|r,with_cached_index_contents rrfrgrtc cs|j}|dr+t|j}|dgD]}t||j }|dur%q|VqdSt |j }|j p4d}| |j ||j }|jpE|}|jD]} tj| ||d}|durXqI|VqIdS)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. r?filesNzutf-8)page_urlbase_url)r/r'r(jsonloadscontentrAr from_jsonr"HTMLLinkParserencodingfeeddecoderanchors from_element) rfrDdatafilelinkparserrr"ranchorr+r+r, parse_linkss*       rT)frozenc@sHeZdZUdZded<ded<ded<ded<d Zd ed <dd dZdS)rgaRepresents one response (or page), along with its URL. :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. bytesrr#r/r%rr"TrVrhr$cCs t|jSr2)rr"rqr+r+r,__str__ s zIndexContent.__str__N)r$r#)r8r9r:__doc____annotations__rhrr+r+r+r,rgs   cs6eZdZdZdfdd Zdd d ZdddZZS)rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. r"r#r$r1cs$tjdd||_d|_g|_dS)NT)convert_charrefs)r3r4r"rr)r5r"r6r+r,r4s zHTMLLinkParser.__init__tagattrslist[tuple[str, str | None]]cCsR|dkr|jdur||}|dur||_dSdS|dkr'|jt|dSdS)Nbasea)rget_hrefrappenddict)r5rrhrefr+r+r,handle_starttags  zHTMLLinkParser.handle_starttagr%cCs"|D] \}}|dkr|SqdS)Nrr+)r5rnamevaluer+r+r,r#s zHTMLLinkParser.get_href)r"r#r$r1)rr#rrr$r1)rrr$r%)r8r9r:rr4rrr;r+r+r6r,rs  rrrreasonstr | ExceptionmethCallable[..., None] | NonecCs|durtj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rYrZ)rrrr+r+r,_handle_get_simple_fail*srrhcCs&t|j}t|j|jd||j|dS)Nr=)rr"rh)rdr@rgrr")r<rhrr+r+r,_make_index_content4s r)rUIndexContent | Nonec Cs|jddd}t|}|rtd||dS|dr;tjt |r;| ds.|d7}t j |d}td|z t|||d }WntyTtd |YdStyp}ztd ||j|jWYd}~dSd}~wttfy}z t||WYd}~dSd}~wttfy}zd |j}t||tjd WYd}~dSd}~wty}zt|d|jWYd}~dSd}~wty}zt|d|jWYd}~dSd}~wty}zt|t|jWYd}~dSd}~wwt||j dS)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.zfile:/z index.htmlz# file: URL is directory, getting %srGrUz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: zproxy connection error: )rh)!r"splitr-rYwarningr(osrPisdirrendswithrKrLurljoinrZr\rFr.r0r/rrrrrcontextinforrrr#rrh)rrGrUr" vcs_schemerSexcrr+r+r,_get_index_contentAsp      rc@seZdZUded<ded<dS)CollectedSourceszSequence[LinkSource | None] find_links index_urlsN)r8r9r:rr+r+r+r,rs  rc@sReZdZdZd#dd Ze d$d%ddZed&ddZ d'd(ddZ d)d!d"Z dS)* LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. rGr search_scoperr$r1cCs||_||_dSr2)rrG)r5rGrr+r+r,r4s zLinkCollector.__init__Foptionsr suppress_no_indexrVcCsd|jg|j}|jr|stdddd|Dg}|jp g}tj|||jd}t ||d}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,css|]}t|VqdSr2r).0r"r+r+r, sz'LinkCollector.create..)rrno_index)rGr) index_urlextra_index_urlsrrYrZr[rrcreater)clsrGrrrrrlink_collectorr+r+r,rs$  zLinkCollector.create list[str]cCs|jjSr2)rrrqr+r+r,rszLinkCollector.find_linksNlocationr package_namer%rcCs4|jj}d|vp|duot||v}t||j|dS)z> Fetch an HTML page containing package links. z:all:Nr)rGrefresh_packager r)r5rrrUshould_force_revalidater+r+r,fetch_responses  zLinkCollector.fetch_response project_namer#candidates_from_pagerrcstfddjD}tfddjD}ttj rKddt ||D}t |ddg|}t d|tt|t|d S) Nc 3(|]}t|jjdddVqdS)Frpage_validator expand_dirrhrNr!rGis_secure_originrlocrrr5r+r,r  z0LinkCollector.collect_sources..c 3r)TrNrrrr+r,rrcSs*g|]}|dur|jdurd|jqS)Nz* )r)rsr+r+r, s  z1LinkCollector.collect_sources..z' location(s) to search for versions of : )rr) collections OrderedDictrget_index_urls_locationsvaluesrrY isEnabledForloggingDEBUG itertoolschainr)rZr[rrz)r5rrindex_url_sourcesfind_links_sourceslinesr+rr,collect_sourcess2        zLinkCollector.collect_sources)rGrrrr$r1F)rGrrr rrVr$r)r$rr2)rrrr%r$r)rr#rrr$r) r8r9r:rr4 classmethodrpropertyrrrr+r+r+r,rs !  r)r"r#r$r%)r<r r$r1)r"r#rGrr$r1r)r"r#rGrrUrVr$r )r@r]r$r%)rwrsr$rsrvr2)rrrrrrr$r1)T)r<r rhrVr$rg)rrrGrrUrVr$r)Tr __future__rr email.messager_r~rrrr urllib.parserKcollections.abcrrrr dataclassesr html.parserroptparser typingr r pip._vendor.packaging.utilsr pip._vendor.requestsr pip._vendor.requests.exceptionsrpip._internal.exceptionsrrrrrrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.utils.urlsrpip._internal.vcsrsourcesrr r! getLoggerr8rYr#r]r- Exceptionr.rErFrTr\rdrersrrrgrrrrrrr+r+r+r,sl                      ;     B