o h9?@sNdZddlmZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlmZmZmZddlmZddlmZddlmZddlmZmZmZdd lmZdd lmZdd l m!Z!m"Z"dd l#m$Z$dd l%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6e 7e8Z9ee:e:fZ;dSddZdTd!d"Z?Gd#d$d$e=Z@dUd'd(ZAdVd)d*ZBdWd-d.ZCGd/d0d0ZDGd1d2d2eZEdXd4d5ZFeFdYd9d:ZGed;d<Gd=d7d7ZHGd>d?d?eZI dZd[dFdGZJ ;d\d]dJdKZKd^dMdNZLGdOdPdPeZMGdQdRdRZNdS)_zO The main purpose of this module is to expose LinkCollector.collect_sources(). ) annotationsN)IterableMutableMappingSequence) dataclass) HTMLParser)Values)Callable NamedTupleProtocol)requests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status)is_archive_fileredact_auth_from_url)vcs)CandidatesFromPage LinkSource build_sourceurlstrreturn str | NonecCs6tjD]}||r|t|dvr|SqdS)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rschemeslower startswithlen)rschemer&P/var/www/html/venv/lib/python3.10/site-packages/pip/_internal/index/collector.py_match_vcs_scheme.s r(cseZdZdfdd ZZS) _NotAPIContent content_typer request_descrNonecst||||_||_dSN)super__init__r*r+)selfr*r+ __class__r&r'r/:s z_NotAPIContent.__init__)r*rr+rrr,)__name__ __module__ __qualname__r/ __classcell__r&r&r1r'r)9sr)responser r,cCs2|jdd}|}|drdSt||jj)z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr"r#r)requestmethod)r7r*content_type_lr&r&r'_ensure_api_header@sr@c@s eZdZdS)_NotHTTPN)r3r4r5r&r&r&r'rAVsrAsessionrcCsFtj|\}}}}}|dvrt|j|dd}t|t|dS)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httphttpsT)allow_redirectsN)urllibparseurlsplitrAheadrr@)rrBr%netlocpathqueryfragmentrespr&r&r'_ensure_api_responseZs  rOcCsxtt|jr t||dtdt||j|dgdddd}t |t |tdt||j d d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. rBzGetting page %sz, )r:z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z max-age=0)Acceptz Cache-Control)r;zFetched page %s as %sr8r9) rrfilenamerOloggerdebugrr<joinrr@r;)rrBrNr&r&r'_get_simple_responsels&  rVr;ResponseHeaderscCs<|rd|vrtj}|d|d<|d}|rt|SdS)z=Determine if we have any encoding information in our headers.r8z content-typecharsetN)emailmessageMessage get_paramr)r;mrXr&r&r'_get_encoding_from_headerss    r^c@s*eZdZdddZdd d Zdd dZdS)CacheablePageContentpage IndexContentrr,cCs|jsJ||_dSr-)cache_link_parsingr`r0r`r&r&r'r/s  zCacheablePageContent.__init__otherobjectboolcCst|t|o|jj|jjkSr-) isinstancetyper`r)r0rdr&r&r'__eq__szCacheablePageContent.__eq__intcCs t|jjSr-)hashr`rr0r&r&r'__hash__s zCacheablePageContent.__hash__N)r`rarr,)rdrerrf)rrj)r3r4r5r/rirmr&r&r&r'r_s  r_c@seZdZdddZdS) ParseLinksr`rarIterable[Link]cCsdSr-r&rcr&r&r'__call__szParseLinks.__call__Nr`rarro)r3r4r5rpr&r&r&r'rnsrnfncs2tjd fdd td fd d }|S)z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. cacheable_pager_r list[Link]cst|jSr-)listr`)rs)rrr&r'wrappersz*with_cached_index_content..wrapperr`racs|jr t|St|Sr-)rbr_ru)r`rrrvr&r'wrapper_wrappers  z2with_cached_index_content..wrapper_wrapperN)rsr_rrt)r`rarrt) functoolscachewraps)rrrxr&rwr'with_cached_index_contents r|r`raroc cs|j}|dr+t|j}|dgD]}t||j }|dur%q|VqdSt |j }|j p4d}| |j ||j }|jpE|}|jD]} tj| ||d}|durXqI|VqIdS)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. r:filesNzutf-8)page_urlbase_url)r*r"r#jsonloadscontentr<r from_jsonrHTMLLinkParserencodingfeeddecoderanchors from_element) r`r?datafilelinkparserrrranchorr&r&r' parse_linkss*       rT)frozenc@sHeZdZUdZded<ded<ded<ded<d Zd ed <dd dZdS)raaRepresents one response (or page), along with its URL. :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. bytesrrr*r rrTrfrbrcCs t|jSr-)rrrlr&r&r'__str__s zIndexContent.__str__N)rr)r3r4r5__doc____annotations__rbrr&r&r&r'ras   cs6eZdZdZdfdd Zdd d ZdddZZS)rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. rrrr,cs$tjdd||_d|_g|_dS)NT)convert_charrefs)r.r/rrr)r0rr1r&r'r/s zHTMLLinkParser.__init__tagattrslist[tuple[str, str | None]]cCsR|dkr|jdur||}|dur||_dSdS|dkr'|jt|dSdS)Nbasea)rget_hrefrappenddict)r0rrhrefr&r&r'handle_starttags  zHTMLLinkParser.handle_starttagr cCs"|D] \}}|dkr|SqdS)Nrr&)r0rnamevaluer&r&r'r!s zHTMLLinkParser.get_href)rrrr,)rrrrrr,)rrrr )r3r4r5rr/rrr6r&r&r1r'r s  rrrreasonstr | ExceptionmethCallable[..., None] | NonecCs|durtj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rSrT)rrrr&r&r'_handle_get_simple_fail(srrbrfcCs&t|j}t|j|jd||j|dS)Nr8)rrrb)r^r;rarr)r7rbrr&r&r'_make_index_content2s rIndexContent | Nonec Cs|jddd}t|}|rtd||dStj|\}}}}}}|dkrHtj tj |rH| ds;|d7}tj|d}td|zt||d }Wnty`td |YdSty|}ztd ||j|jWYd}~dSd}~wty}z t||WYd}~dSd}~wty}z t||WYd}~dSd}~wty}zd } | t|7} t|| tjd WYd}~dSd}~wtjy}zt|d|WYd}~dSd}~wtjyt|dYdSwt||jdS)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.r/z index.htmlz# file: URL is directory, getting %srPz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)rb) rsplitr(rSwarningrFrGurlparseosrKisdirr= url2pathnameendswithurljoinrTrVrAr)r+r*rrrrrinfor ConnectionErrorTimeoutrrb) rrBr vcs_schemer%_rKrNexcrr&r&r'_get_index_content?sn       rc@seZdZUded<ded<dS)CollectedSourceszSequence[LinkSource | None] find_links index_urlsN)r3r4r5rr&r&r&r'r|s  rc@sNeZdZdZd!dd Ze d"d#ddZed$ddZd%ddZ d&ddZ d S)' LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. rBr search_scoperrr,cCs||_||_dSr-)rrB)r0rBrr&r&r'r/s zLinkCollector.__init__Foptionsrsuppress_no_indexrfcCsd|jg|j}|jr|stdddd|Dg}|jp g}tj|||jd}t ||d}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,css|]}t|VqdSr-r).0rr&r&r' sz'LinkCollector.create..)rrno_index)rBr) index_urlextra_index_urlsrrSrTrUrrcreater)clsrBrrrrrlink_collectorr&r&r'rs$  zLinkCollector.create list[str]cCs|jjSr-)rrrlr&r&r'rszLinkCollector.find_linkslocationrrcCst||jdS)z> Fetch an HTML page containing package links. rP)rrB)r0rr&r&r'fetch_responseszLinkCollector.fetch_response project_namercandidates_from_pagerrcstfddjD}tfddjD}ttj rKddt ||D}t |ddg|}t d|tt|t|d S) Nc 3(|]}t|jjdddVqdS)Frpage_validator expand_dirrbrNrrBis_secure_originrlocrrr0r&r'r  z0LinkCollector.collect_sources..c 3r)TrNrrrr&r'rrcSs*g|]}|dur|jdurd|jqS)Nz* )r)rsr&r&r' s  z1LinkCollector.collect_sources..z' location(s) to search for versions of : )rr) collections OrderedDictrget_index_urls_locationsvaluesrrS isEnabledForloggingDEBUG itertoolschainr$rTrUrru)r0rrindex_url_sourcesfind_links_sourceslinesr&rr'collect_sourcess2        zLinkCollector.collect_sourcesN)rBrrrrr,)F)rBrrrrrfrr)rr)rrrr)rrrrrr) r3r4r5rr/ classmethodrpropertyrrrr&r&r&r'rs !  r)rrrr )r7r rr,)rrrBrrr,)rrrBrrr )r;rWrr )rrrnrrnrqr-)rrrrrrrr,)T)r7r rbrfrra)rrrBrrr)Or __future__rr email.messagerYryrrrr urllib.parserFurllib.requestcollections.abcrrr dataclassesr html.parserroptparsertypingr r r pip._vendorr pip._vendor.requestsr pip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.vcsrsourcesrrr getLoggerr3rSrrWr( Exceptionr)r@rArOrVr^r_rnr|rrarrrrrrr&r&r&r'sh                     ?     =