hidZddlmZddlZddlZddlZddlmZm Z ddl m Z m Z ddl mZddlZddlZddlmZmZdd lmZmZmZdd lmZej4j7d Zd Ze d GddZGddZeZ GddZ!ee jD d ddZ#ee jHdZ$GddZ%ddZ&y)ai`tldextract` accurately separates a URL's subdomain, domain, and public suffix. It does this via the Public Suffix List (PSL). >>> import tldextract >>> tldextract.extract("http://forums.news.cnn.com/") ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False) >>> tldextract.extract("http://forums.bbc.co.uk/") # United Kingdom ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False) >>> tldextract.extract("http://www.worldbank.org.kg/") # Kyrgyzstan ExtractResult(subdomain='www', domain='worldbank', suffix='org.kg', is_private=False) Note subdomain and suffix are _optional_. Not all URL-like inputs have a subdomain or a valid suffix. >>> tldextract.extract("google.com") ExtractResult(subdomain='', domain='google', suffix='com', is_private=False) >>> tldextract.extract("google.notavalidsuffix") ExtractResult(subdomain='google', domain='notavalidsuffix', suffix='', is_private=False) >>> tldextract.extract("http://127.0.0.1:8080/deployed/") ExtractResult(subdomain='', domain='127.0.0.1', suffix='', is_private=False) To rejoin the original hostname, if it was indeed a valid, registered hostname: >>> ext = tldextract.extract("http://forums.bbc.co.uk") >>> ext.top_domain_under_public_suffix 'bbc.co.uk' >>> ext.fqdn 'forums.bbc.co.uk' ) annotationsN) CollectionSequence) dataclassfield)wraps) DiskCache get_cache_dir)lenient_netloc looks_like_iplooks_like_ipv6)get_suffix_listsTLDEXTRACT_CACHE_TIMEOUT)z4https://publicsuffix.org/list/public_suffix_list.datzQhttps://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.datT)orderceZdZUdZded< ded< ded< ded< ed Zded < edd Zedd Z edd Z eddZ eddZ eddZ eddZy) ExtractResultaA URL's extracted subdomain, domain, and suffix. These first 3 fields are what most users of this library will care about. They are the split, non-overlapping hostname components of the input URL. They can be used to rebuild the original URL's hostname. Beyond the first 3 fields, the class contains metadata fields, like a flag that indicates if the input URL's suffix is from a private domain. str subdomaindomainsuffixbool is_privateF)reprregistry_suffixc|jrQ|js |jr9djd|j|j|jfDSy)aThe Fully Qualified Domain Name (FQDN), if there is a proper `domain` and `suffix`, or else the empty string. >>> extract("http://forums.bbc.co.uk/path/to/file").fqdn 'forums.bbc.co.uk' >>> extract("http://localhost:8080").fqdn '' .c3&K|] }|s| ywN).0is X/var/www/html/ranktracker/api/venv/lib/python3.12/site-packages/tldextract/tldextract.py z%ExtractResult.fqdn..ysW!UVAWs)rrrjoinrselfs r#fqdnzExtractResult.fqdnosB ;;DKK4??88W T[['QWW Wc|jr9|js-|js!t|jr |jSy)a,The IPv4 address, if that is what the input domain/URL was, or else the empty string. >>> extract("http://127.0.0.1/path/to/file").ipv4 '127.0.0.1' >>> extract("http://127.0.0.1.1/path/to/file").ipv4 '' >>> extract("http://256.1.1.1").ipv4 '' r%)rrrr r's r#ipv4zExtractResult.ipv4|s1 KK[[DNNdkk*;; r*cd}t|j|k\rX|jddk(rF|jddk(r4|js(|js|jdd}t |r|Sy)aThe IPv6 address, if that is what the input domain/URL was, or else the empty string. >>> extract( ... "http://[aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1]/path/to/file" ... ).ipv6 'aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1' >>> extract( ... "http://[aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1.1]/path/to/file" ... ).ipv6 '' >>> extract("http://[aBcD:ef01:2345:6789:aBcD:ef01:256.0.0.1]").ipv6 '' r[]r r%)lenrrrr)r(min_num_ipv6_chars debracketeds r#ipv6zExtractResult.ipv6sj   2 2 A#% B3&[[DNN++a+K{+""r*cRtjdtd|jS)aThe `domain` and `suffix` fields joined with a dot, if they're both set, or else the empty string. >>> extract("http://forums.bbc.co.uk").registered_domain 'bbc.co.uk' >>> extract("http://localhost:8080").registered_domain '' .. deprecated:: 6.0.0 This property is deprecated and will be removed in the next major version. Use `top_domain_under_public_suffix` instead, which has the same behavior but a more accurate name. This is an alias for the `top_domain_under_public_suffix` property. `registered_domain` is so called because is roughly the domain the owner paid to register with a registrar or, in the case of a private domain, "registered" with the domain owner. If the input was not something one could register, this property returns the empty string. To distinguish the case of private domains, consider Blogspot, which is in the PSL's private domains. If `include_psl_private_domains` was set to `False`, the `registered_domain` property of a Blogspot URL represents the domain the owner of Blogspot registered with a registrar, i.e. Google registered "blogspot.com". If `include_psl_private_domains=True`, the `registered_domain` property represents the "blogspot.com" _subdomain_ the owner of a blog "registered" with Blogspot. >>> extract( ... "http://waiterrant.blogspot.com", include_psl_private_domains=False ... ).registered_domain 'blogspot.com' >>> extract( ... "http://waiterrant.blogspot.com", include_psl_private_domains=True ... ).registered_domain 'waiterrant.blogspot.com' To always get the same joined string, regardless of the `include_psl_private_domains` setting, consider the `top_domain_under_registry_suffix` property. zThe 'registered_domain' property is deprecated and will be removed in the next major version. Use 'top_domain_under_public_suffix' instead, which has the same behavior but a more accurate name.) stacklevel)warningswarnDeprecationWarningtop_domain_under_public_suffixr's r#registered_domainzExtractResult.registered_domains+T   r   222r*c|j|jg}|jr3|jt |jj ddj |S)aXThe domain name in Reverse Domain Name Notation. Joins extracted components of the input URL in reverse domain name notation. The suffix is used as the leftmost component, followed by the domain, then followed by the subdomain with its parts reversed. Reverse Domain Name Notation is typically used to organize namespaces for packages and plugins. Technically, a full reversal would reverse the parts of the suffix, e.g. "co.uk" would become "uk.co", but this is not done in practice when Reverse Domain Name Notation is called for. So this property leaves the `suffix` part in its original order. >>> extract("login.example.com").reverse_domain_name 'com.example.login' >>> extract("login.example.co.uk").reverse_domain_name 'co.uk.example.login' r)rrrextendreversedsplitr&)r(stacks r#reverse_domain_namez!ExtractResult.reverse_domain_namesK(dkk* >> LL$.."6"6s";< =xxr*c|j}|r |js|S|jjddz}dj |j d| dS)aThe rightmost domain label and `registry_suffix` joined with a dot, if such a domain is available and `registry_suffix` is set, or else the empty string. The rightmost domain label might be in the `domain` field, or, if the input URL's suffix is a PSL private domain, in the public suffix `suffix` field. If the input was not in the PSL's private domains, this property is equivalent to `top_domain_under_public_suffix`. >>> extract( ... "http://waiterrant.blogspot.com", include_psl_private_domains=False ... ).top_domain_under_registry_suffix 'blogspot.com' >>> extract( ... "http://waiterrant.blogspot.com", include_psl_private_domains=True ... ).top_domain_under_registry_suffix 'blogspot.com' >>> extract("http://localhost:8080").top_domain_under_registry_suffix '' rr7N)r<rrcountr&rA)r(r< num_labelss r# top_domain_under_registry_suffixz.ExtractResult.top_domain_under_registry_suffixs^,*.)L)L&-T__1 1))//4q8 xx6<>> extract("http://forums.bbc.co.uk").top_domain_under_public_suffix 'bbc.co.uk' >>> extract("http://localhost:8080").top_domain_under_public_suffix '' rr%)rrr's r#r<z,ExtractResult.top_domain_under_public_suffixs- ;;4;;kk]!DKK=1 1r*N)returnr)__name__ __module__ __qualname____doc____annotations__rrpropertyr)r,r5r=rCrGr<r r*r#rr=sN| K K !e,OS,  $4/3/3b0QQ8  r*rceZdZdZeedddef ddZ d ddZ d ddZ d dd Z d dd Z d dd Z e ddd Z d dd Zy) TLDExtractzOA callable for extracting, subdomain, domain, and suffix components from a URL.TFr c$|xsd}td|D|_||_|js|s|js td||_||_d|_t|tr t|n||_ t||_ y)aConstruct a callable for extracting subdomain, domain, and suffix components from a URL. Upon calling it, it first checks for a JSON in `cache_dir`. By default, the `cache_dir` will live in the tldextract directory. You can disable the caching functionality of this module by setting `cache_dir` to `None`. If the cached version does not exist, such as on the first run, HTTP request the URLs in `suffix_list_urls` in order, and use the first successful response for public suffix definitions. Subsequent, untried URLs are ignored. The default URLs are the latest version of the Mozilla Public Suffix List and its mirror, but any similar document URL could be specified. Local files can be specified by using the `file://` protocol (see `urllib2` documentation). To disable HTTP requests, set this to an empty sequence. If there is no cached version loaded and no data is found from the `suffix_list_urls`, the module will fall back to the included TLD set snapshot. If you do not want this behavior, you may set `fallback_to_snapshot` to False, and an exception will be raised instead. The Public Suffix List includes a list of "private domains" as TLDs, such as blogspot.com. These do not fit `tldextract`'s definition of a suffix, so these domains are excluded by default. If you'd like them included instead, set `include_psl_private_domains` to True. You can specify additional suffixes in the `extra_suffixes` argument. These will be merged into whatever public suffix definitions are already in use by `tldextract`, above. cache_fetch_timeout is passed unmodified to the underlying request object per the requests documentation here: http://docs.python-requests.org/en/master/user/advanced/#timeouts cache_fetch_timeout can also be set to a single value with the environment variable TLDEXTRACT_CACHE_TIMEOUT, like so: TLDEXTRACT_CACHE_TIMEOUT="1.2" When set this way, the same timeout value will be used for both connect and read timeouts r c3^K|]%}|js|j'ywr)strip)r!urls r#r$z&TLDExtract.__init__..Ws#& syy{CIIK& s--zThe arguments you have provided disable all ways for tldextract to obtain data. Please provide a suffix list data, a cache_dir, or set `fallback_to_snapshot` to `True`.N) tuplesuffix_list_urlsfallback_to_snapshot ValueErrorinclude_psl_private_domainsextra_suffixes _extractor isinstancerfloatcache_fetch_timeoutr _cache)r( cache_dirrWrXrZr[r_s r#__init__zTLDExtract.__init__$sd,1r %& #3& ! %9!%%d6O6O;  ,G(,@D-s3 % &$   * r*Nc*|j|||S)zAlias for `extract_str`.session) extract_strr(rUrZres r#__call__zTLDExtract.__call__ns%@'RRr*c<|jt|||S)aTake a string URL and splits it into its subdomain, domain, and suffix components. I.e. its effective TLD, gTLD, ccTLD, etc. components. >>> extractor = TLDExtract() >>> extractor.extract_str("http://forums.news.cnn.com/") ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False) >>> extractor.extract_str("http://forums.bbc.co.uk/") ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False) Allows configuring the HTTP request via the optional `session` parameter. For example, if you need to use a HTTP proxy. See also `requests.Session`. >>> import requests >>> session = requests.Session() >>> # customize your session here >>> with session: ... extractor.extract_str("http://forums.news.cnn.com/", session=session) ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False) rd)_extract_netlocr rgs r#rfzTLDExtract.extract_strws)6## 3 !|j|j||S)aTake the output of urllib.parse URL parsing methods and further splits the parsed URL. Splits the parsed URL into its subdomain, domain, and suffix components, i.e. its effective TLD, gTLD, ccTLD, etc. components. This method is like `extract_str` but faster, as the string's domain name has already been parsed. >>> extractor = TLDExtract() >>> extractor.extract_urllib( ... urllib.parse.urlsplit("http://forums.news.cnn.com/") ... ) ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False) >>> extractor.extract_urllib(urllib.parse.urlsplit("http://forums.bbc.co.uk/")) ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False) rd)rjnetlocrgs r#extract_urllibzTLDExtract.extract_urllibs(,## JJ3W$  r*c|jddjddjdd}d}t||k\r.|ddk(r&|dd k(rt|d drtd |d d d S|j d}|j |j ||}d}|s)t||k(rt|rtd |d d d S|s%tdj|dd|dd d d S|\\} } \} } | dk\rdj|d| d z nd } | dkDr|| d z nd }dj|| d}| jrdj|| dn|}t| ||| j|S)Nu。ru．u｡r.rr/r0r1r r%F)rr)rZ)rrrrrr7) replacer2rrrA_get_tld_extractor suffix_indexr r&r)r(rlrZrenetloc_with_ascii_dotsr3labels maybe_indexesnum_ipv4_labelspublic_suffix_indexpublic_suffix_noderegistry_suffix_indexregistry_suffix_noderr public_suffixrs r#rjzTLDExtract._extract_netlocs NN8X . WXx ( WXx (   & '+= =&q)S0&r*c1 6q <= *B5RT (--c2//8EE 0KF F .45 *B5RT  ((6#2;/bz "    5 "4 9 "$8 #a' HHV51A56 7  5H!4K+a/0QS(;( ",, HHV123 4   )44+   r*cpd|_|jj|r|j|yy)z/Force fetch the latest suffix list definitions.Nrd)r\r`clearrp)r( fetch_nowres r#updatezTLDExtract.updates4    # #G # 4 r*cTt|j|jS)zThe list of TLDs used by default. This will vary based on `include_psl_private_domains` and `extra_suffixes`. rd)listrptlds)r(res r#rzTLDExtract.tldss& D++G+<AACDDr*ch|jr |jSt|j|j|j|j |\}}t |||jgs tdt||t|j|j|_|jS)a1Get or compute this object's TLDExtractor. Looks up the TLDExtractor in roughly the following order, based on the settings passed to __init__: 1. Memoized on `self` 2. Local system _cache file 3. Remote PSL, over HTTP 4. Bundled PSL snapshot file )cacheurlsr_rXrez)No tlds set. Cannot proceed without tlds.) public_tlds private_tlds extra_tldsrZ) r\rr`rWr_rXanyr[rY_PublicSuffixListTLDExtractorrrZ)r(rerrs r#rpzTLDExtract._get_tld_extractor s ???? "$4++&& $ 8 8!%!:!: % ! \Kt/B/BCDHI I7#%D//0(,(H(H   r*)raz str | NonerW Sequence[str]rXrrZrr[rr_zstr | float | NonerINone)NNrUrrZ bool | Nonererequests.Session | NonerIr)rUz3urllib.parse.ParseResult | urllib.parse.SplitResultrZrrerrIrr)rlrrZrrerrIrFN)r}rrerrIr)rerrI list[str])rerrIr)rJrKrLrMr PUBLIC_SUFFIX_LIST_URLS CACHE_TIMEOUTrbrhrfrmrjr~rOrrpr r*r#rQrQ sY !.*A%),1(*2?H+H+(H+# H+ &* H+ & H+0H+ H+Z48+/ S S&1S) S  S48+/   &1 )    D48+/  @ &1 )    <,0 F F &1F ) F  F RKO550G5 5EE26!.! &!r*rQcXeZdZdZ d ddZe d d dZd d dZy) Triez:Trie for storing eTLDs with their labels in reverse-order.Nc6|r|ni|_||_||_y)zTODO.N)matchesendr)r(rrrs r#rbz Trie.__init__3s#*wr $r*ct}|D]}|j||g}|D]}|j|d|S)z?Create a Trie from a list of suffixes and return its root node.T)r add_suffix)public_suffixesprivate_suffixes root_noders r#createz Trie.create>s] F % )F   ( )  #! & /F   . /r*c|}|jd}|j|D]6}||jvrt|j|<|j|}8d|_||_y)z+Append a suffix's labels to this Trie node.rTN)rAreverserrrr)r(rrnoderslabels r#rzTrie.add_suffixQshc" 'EDLL(&*f U#<<&D ' $r*)NFF)rzdict[str, Trie] | NonerrrrrIrr)rzCollection[str]rzCollection[str] | NonerIrF)rrrrrIr)rJrKrLrMrb staticmethodrrr r*r#rr0ssD+/ %' % % %  %48(0 $ %r*rct|||S)N)rZre) TLD_EXTRACTOR)rUrZres r#extractras  )Dg r*c,tj|i|Sr)rr~)argskwargss r#r~r~ls    0 00r*cJeZdZdZ d ddZdd dZ d d dZy) rz8Wrapper around this project's main algo for PSL lookups.c$||_||_||_t||z|z|_t||z|_t j|j t||_t j|j |_ yr) rZrr frozensettlds_incl_privatetlds_excl_privaterrtlds_incl_private_trietlds_excl_private_trie)r(rrrrZs r#rbz&_PublicSuffixListTLDExtractor.__init__ts,G(&(!*;+E +R!S!*;+C!D&*kk  " "Il$;' #'+kk$2H2H&I#r*NcR| |j}|r |jS|jS)z,Get the currently filtered list of suffixes.)rZrr)r(rZs r#rz"_PublicSuffixListTLDExtractor.tldss9 & .*.*J*J '+  " " '' r*c| |j}|r |jn |jx}}t|x}x}}t |D]}t |} | |j vr3|dz}|j | }|jr|}|js|}|}Od|j v} | r/d| z|j v} | r|n|dz |j df||ffcSn|t|k(ry||f||ffS)zReturn the index of the first public suffix label, the index of the first registry suffix label, and their corresponding trie nodes. Returns `None` if no suffix is found. Nr *!) rZrrr2r@_decode_punycoderrr) r(splrZrreg_node suffix_idxreg_idx label_idxr decoded_label is_wildcardis_wildcard_exceptions r#rqz*_PublicSuffixListTLDExtractor.suffix_indexs0 ' .*.*J*J '+  ' ',, x ,/s83 3Wyc] E,U3M ,Q ||M288!*J??#'"+-K(+m(;t||(K%!6IIMLL% / 2 S !T"Wh$788r*r)rrrrrrrZrr)rZrrIzfrozenset[str])rrrZrrIz0tuple[tuple[int, Trie], tuple[int, Trie]] | None)rJrKrLrMrbrrqr r*r#rrqscB-2 JJ J J &* J$  JN,9,9;F,9 9,9r*rc|j}|jd}|r tj|S|S#tt f$rY|SwxYw)Nzxn--)lower startswithidnadecode UnicodeError IndexError)rloweredlooks_like_punys r#rrsYkkmG((0O ;;w' ' Nj)   N s<AArr)rrrIr)'rM __future__ros urllib.parseurllibr9collections.abcrr dataclassesrr functoolsrrrequestsrr r remoter r r suffix_listrenvirongetrrrrQrrrhrr~rrr r*r#rs"H# 0( +BB) 9:   ___DJJZ .%.%b}05'+ !,% }11L9L9^r*