+
    öÙ†j}&  ã                   ó¦   € R t ^ RIt^ RIt^ RIt^ RIt^ RItR.t]P                  ! RR4      t	R t
R t ! R R4      t ! R R	4      t ! R
 R4      tR# )a  robotparser.py

Copyright (C) 2000  Bastian Kleineidam

You can choose between two licenses when using this package:
1) GNU GPLv2
2) PSF license for Python 2.2

The robots.txt Exclusion Protocol is implemented as specified in
http://www.robotstxt.org/norobots-rfc.txt
NÚRobotFileParserÚRequestRatezrequests secondsc                 ó†   € \         P                  P                  V R R7      p\         P                  P                  VR R7      # )Úsurrogateescape)Úerrors)ÚurllibÚparseÚunquoteÚquote)ÚpathÚunquoteds   & Ú)/usr/lib/python3.14/urllib/robotparser.pyÚ	normalizer      s7   € Ü�|‰|×#Ñ# DÐ1BÐ#ÓC€HÜ�<‰<×Ñ˜hÐ/@ÐÓAÐAó    c                 ó¦   € V P                  R 4      w  rp\        V 4      p V'       d*   \        P                  ! RR V4      pV R V,           ,          p V # )Ú?z[^=&]+c                 ó&   € \        V ^ ,          4      # )é    )r   )Úms   &r   Ú<lambda>Ú normalize_path.<locals>.<lambda>    s   € ¬I°a¸µd¬Or   )Ú	partitionr   ÚreÚsub)r   ÚsepÚquerys   &  r   Únormalize_pathr      sG   € Ø—~‘~ cÓ*Ñ€DˆuÜ�T‹?€Dß
Ü—’�yÑ";¸UÓCˆØ��e•ÕˆØ€Kr   c                   óp   a € ] tR t^%t o RtRR ltR tR tR tR t	R t
R tR	 tR
 tR tR tR tRtV tR# )r   zjThis class provides a set of methods to read, parse and answer
questions about a single robots.txt file.

c                ó|   € . V n         . V n        R V n        RV n        RV n        V P                  V4       ^ V n        R # )NF)ÚentriesÚsitemapsÚdefault_entryÚdisallow_allÚ	allow_allÚset_urlÚlast_checked©ÚselfÚurls   &&r   Ú__init__ÚRobotFileParser.__init__+   s;   € ØˆŒØˆŒØ!ˆÔØ!ˆÔØˆŒØ�‰�SÔØˆÖr   c                ó   € V P                   # )zŸReturns the time the robots.txt file was last fetched.

This is useful for long-running web spiders that need to
check for new robots.txt files periodically.

)r%   ©r'   s   &r   ÚmtimeÚRobotFileParser.mtime4   s   € ð × Ñ Ð r   c                ó8   € ^ RI pVP                  4       V n        R# )zISets the time the robots.txt file was last fetched to the
current time.

N)Útimer%   )r'   r0   s   & r   ÚmodifiedÚRobotFileParser.modified=   s   € ó
 	Ø ŸI™I›KˆÖr   c                óx   € Wn         \        P                  P                  V4      R,          w  V n        V n        R# )z,Sets the URL referring to a robots.txt file.:é   é   NN)r(   r   r   ÚurlsplitÚhostr   r&   s   &&r   r$   ÚRobotFileParser.set_urlE   s)   € àŒÜ%Ÿ|™|×4Ñ4°SÓ9¸#Õ>ÑˆŒ	�4–9r   c                óÖ  €  \         P                  P                  V P                  4      pVP	                  4       pV P                  VP                  RR4      P                  4       4       R#   \         P                  P                   d^   pTP                  R9   d	   RT n        M)TP                  R8¼  d   TP                  R8  d   RT n        TP                  4         Rp?R# Rp?ii ; i)z4Reads the robots.txt URL and feeds it to the parser.zutf-8r   Ti�  iô  N)i‘  i“  )r   ÚrequestÚurlopenr(   Úreadr   ÚdecodeÚ
splitlinesÚerrorÚ	HTTPErrorÚcoder"   r#   Úclose)r'   ÚfÚrawÚerrs   &   r   r<   ÚRobotFileParser.readJ   s¥   € ð
	LÜ—‘×&Ñ& t§x¡xÓ0ˆAð —&‘&“(ˆCØ�J‰J�s—z‘z 'Ð+<Ó=×HÑHÓJÖKøô �|‰|×%Ñ%ô 	Ø�x‰x˜:Ô%Ø$(�Õ!Ø—‘˜S” S§X¡X°¤^Ø!%�”Ø�I‰I�KŠKûð	ús   ‚)A, Á,C(ÂAC#Ã#C(c                óŽ   € R VP                   9   d   V P                  f	   Wn        R# R# V P                  P                  V4       R# )Ú*N)Ú
useragentsr!   r   Úappend)r'   Úentrys   &&r   Ú
_add_entryÚRobotFileParser._add_entryX   s;   € Ø�%×"Ñ"Ô"à×!Ñ!Ò)à%*Ö"ñ *ð �L‰L×Ñ Ö&r   c                óÐ  € ^ p\        4       pV P                  4        V EF(  pV'       g9   V^8X  d   \        4       p^ pM$V^8X  d   V P                  V4       \        4       p^ pVP                  R4      pV^ 8¼  d   VRV pVP	                  4       pV'       g   K{  VP                  R^4      p\        V4      ^8X  g   KŸ  V^ ,          P	                  4       P                  4       V^ &   V^,          P	                  4       V^&   V^ ,          R8X  dJ   V^8X  d   V P                  V4       \        4       pVP                  P                  V^,          4       ^pEK8  V^ ,          R8X  d<   V^ 8w  d2   VP                  P                  \        V^,          R4      4       ^pEK~  EK�  V^ ,          R8X  d<   V^ 8w  d2   VP                  P                  \        V^,          R4      4       ^pEKÇ  EKÊ  V^ ,          R	8X  dR   V^ 8w  dH   V^,          P	                  4       P                  4       '       d   \        V^,          4      Vn        ^pEK&  EK)  V^ ,          R
8X  d¿   V^ 8w  dµ   V^,          P                  R4      p\        V4      ^8X  dˆ   V^ ,          P	                  4       P                  4       '       d]   V^,          P	                  4       P                  4       '       d2   \        \        V^ ,          4      \        V^,          4      4      Vn        ^pEKò  EKõ  V^ ,          R8X  g   EK  V P"                  P                  V^,          4       EK+  	  V^8X  d   V P                  V4       R# R# )z|Parse the input lines from a robots.txt file.

We allow that a user-agent: line is not preceded by
one or more blank lines.
Ú#NÚ:z
user-agentÚdisallowFÚallowTzcrawl-delayzrequest-rateÚ/Úsitemap)ÚEntryr1   rL   ÚfindÚstripÚsplitÚlenÚlowerrI   rJ   Ú	rulelinesÚRuleLineÚisdigitÚintÚdelayr   Úreq_rater    )r'   ÚlinesÚstaterK   ÚlineÚiÚnumberss   &&     r   r   ÚRobotFileParser.parsea   s©  € ð ˆÜ“ˆà�‰ŒÜˆDßØ˜A”:Ü!›G�EØ‘EØ˜a”ZØ—O‘O EÔ*Ü!›G�EØ�Eà—	‘	˜#“ˆAØ�AŒvØ˜B˜Q�x�Ø—:‘:“<ˆDßÙØ—:‘:˜c 1Ó%ˆDÜ�4‹y˜AŽ~Ø˜q�'Ÿ-™-›/×/Ñ/Ó1��Q‘Ø˜q�'Ÿ-™-›/��Q‘Ø˜•7˜lÔ*Ø ”zØŸ™¨Ô.Ü %£˜Ø×$Ñ$×+Ñ+¨D°­GÔ4Ø“EØ˜!•W 
Ô*Ø ”zØŸ™×.Ñ.¬x¸¸Q½ÀÓ/GÔHØ !›ò "ð ˜!•W Ô'Ø ”zØŸ™×.Ñ.¬x¸¸Q½ÀÓ/FÔGØ !›ò "ð ˜!•W Ô-Ø ”zð   �7Ÿ=™=›?×2Ñ2×4Ò4Ü*-¨d°1­g«,˜EœKØ !›ò "ð ˜!•W Ô.Ø ”zØ"& q¥'§-¡-°Ó"4˜ä ›L¨AÔ-°'¸!µ*×2BÑ2BÓ2D×2LÑ2L×2NÒ2NØ '¨¥
× 0Ñ 0Ó 2× :Ñ :× <Ò <Ü-8¼¸WÀQ½Z»Ì#ÈgÐVWÍjË/Ó-Z˜EœNØ !›ò "ð ˜!•W 	×)ð
 —M‘M×(Ñ(¨¨a­×1ño ðp �AŒ:Ø�O‰O˜EÖ"ñ r   c                ó  € V P                   '       d   R# V P                  '       d   R# V P                  '       g   R# \        P                  P                  VR4      p\        P                  P                  ! RR.VR,          O5!  p\        V4      pV'       g   RpV P                   F-  pVP                  V4      '       g   K  VP                  V4      u # 	  V P                  '       d   V P                  P                  V4      # R# )z=using the parsed robots.txt decide if useragent can fetch urlFTÚ N:é   NNrS   )r"   r#   r%   r   r   Ú	_urlsplitÚ_urlunsplitr   r   Ú
applies_toÚ	allowancer!   )r'   Ú	useragentr(   Ú
parsed_urlrK   s   &&&  r   Ú	can_fetchÚRobotFileParser.can_fetchª   sÍ   € à××ÐÙØ�>�>ˆ>Ùð
 × × Ð Ùô
 —\‘\×+Ñ+¨C°Ó4ˆ
Ü�l‰l×&Ò& t¨TÐC°J¸rµNÓCˆÜ˜SÓ!ˆßØˆCØ—\”\ˆEØ×Ñ 	×*Ô*Ø—‘ sÓ+Ò+ñ "ð ××ÐØ×%Ñ%×/Ñ/°Ó4Ð4ár   c                óö   € V P                  4       '       g   R # V P                   F(  pVP                  V4      '       g   K  VP                  u # 	  V P                  '       d   V P                  P                  # R # ©N)r-   r   rl   r_   r!   ©r'   rn   rK   s   && r   Úcrawl_delayÚRobotFileParser.crawl_delayÈ   s\   € Ø�z‰z�|Š|ÙØ—\”\ˆEØ×Ñ 	×*Ô*Ø—{‘{Ò"ñ "ð ××ÐØ×%Ñ%×+Ñ+Ð+Ùr   c                óö   € V P                  4       '       g   R # V P                   F(  pVP                  V4      '       g   K  VP                  u # 	  V P                  '       d   V P                  P                  # R # rs   )r-   r   rl   r`   r!   rt   s   && r   Úrequest_rateÚRobotFileParser.request_rateÒ   s\   € Ø�z‰z�|Š|ÙØ—\”\ˆEØ×Ñ 	×*Ô*Ø—~‘~Ò%ñ "ð ××ÐØ×%Ñ%×.Ñ.Ð.Ùr   c                óB   € V P                   '       g   R # V P                   # rs   )r    r,   s   &r   Ú	site_mapsÚRobotFileParser.site_mapsÜ   s   € Ø�}�}ˆ}ÙØ�}‰}Ðr   c                óš   € V P                   pV P                  e   WP                  .,           pRP                  \        \        V4      4      # )Nz

)r   r!   ÚjoinÚmapÚstr)r'   r   s   & r   Ú__str__ÚRobotFileParser.__str__á   s>   € Ø—,‘,ˆØ×ÑÒ)Ø×!3Ñ!3Ð 4Õ4ˆGØ�{‰{œ3œs GÓ,Ó-Ð-r   )	r#   r!   r"   r   r7   r%   r   r    r(   N)rh   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r)   r-   r1   r$   r<   rL   r   rp   ru   rx   r{   r�   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r   r   r   %   sN   ø‡ € ñô
ò!ò(ò?ò
Lò'òG#òRò<òò÷
.ð .r   c                   ó6   a € ] tR t^çt o RtR tR tR tRtV t	R# )r\   zhA rule line is a single "Allow:" (allowance==True) or "Disallow:"
(allowance==False) followed by a path.c                óT   € VR 8X  d   V'       g   Rp\        V4      V n        W n        R# )rh   TN)r   r   rm   )r'   r   rm   s   &&&r   r)   ÚRuleLine.__init__ê   s!   € Ø�2Œ:ŸiàˆIÜ" 4Ó(ˆŒ	Ø"Žr   c                óf   € V P                   R 8H  ;'       g    VP                  V P                   4      # )rH   )r   Ú
startswith)r'   Úfilenames   &&r   rl   ÚRuleLine.applies_toñ   s(   € Ø�y‰y˜CÑ×AÐA 8×#6Ñ#6°t·y±yÓ#AÐAr   c                ó^   € V P                   '       d   R MRR,           V P                  ,           # )ÚAllowÚDisallowz: ©rm   r   r,   s   &r   r�   ÚRuleLine.__str__ô   s    € ØŸ>Ÿ>˜>‘¨z¸TÕAÀDÇIÁIÕMÐMr   r–   N)
rƒ   r„   r…   r†   r‡   r)   rl   r�   rˆ   r‰   rŠ   s   @r   r\   r\   ç   s!   ø‡ € ñ1ò#òB÷Nð Nr   r\   c                   ó<   a € ] tR t^øt o RtR tR tR tR tRt	V t
R# )rU   z?An entry has one or more user-agents and zero or more rulelinesc                ó>   € . V n         . V n        R V n        R V n        R # rs   )rI   r[   r_   r`   r,   s   &r   r)   ÚEntry.__init__ú   s   € ØˆŒØˆŒØˆŒ
ØˆŽr   c                óª  € . pV P                    F  pVP                  R V 24       K  	  V P                  e   VP                  RV P                   24       V P                  e8   V P                  pVP                  RVP                   RVP
                   24       VP                  \        \        V P                  4      4       RP                  V4      # )zUser-agent: zCrawl-delay: zRequest-rate: rS   Ú
)rI   rJ   r_   r`   ÚrequestsÚsecondsÚextendr   r€   r[   r~   )r'   ÚretÚagentÚrates   &   r   r�   ÚEntry.__str__   s¢   € ØˆØ—_”_ˆEØ�J‰J˜ e WÐ-Ö.ñ %à�:‰:Ò!Ø�J‰J˜ t§z¡z lÐ3Ô4Ø�=‰=Ò$Ø—=‘=ˆDØ�J‰J˜¨¯© °a¸¿¹°~ÐFÔGØ�
‰
”3”s˜DŸN™NÓ+Ô,Ø�y‰y˜‹~Ðr   c                ó¾   € VP                  R4      ^ ,          P                  4       pV P                   F&  pVR8X  d    R# VP                  4       pW!9   g   K%   R# 	  R# )z2check if this entry applies to the specified agentrS   rH   TF)rX   rZ   rI   )r'   rn   r¡   s   && r   rl   ÚEntry.applies_to  sQ   € ð —O‘O CÓ(¨Õ+×1Ñ1Ó3ˆ	Ø—_”_ˆEØ˜Œ|âØ—K‘K“MˆEØÖ!Úñ %ñ r   c                óv   € V P                    F(  pVP                  V4      '       g   K  VP                  u # 	  R# )zJPreconditions:
- our agent applies to this entry
- filename is URL encodedT)r[   rl   rm   )r'   r‘   rc   s   && r   rm   ÚEntry.allowance  s0   € ð —N”NˆDØ�‰˜x×(Ô(Ø—~‘~Ò%ñ #ñ r   )r_   r`   r[   rI   N)rƒ   r„   r…   r†   r‡   r)   r�   rl   rm   rˆ   r‰   rŠ   s   @r   rU   rU   ø   s!   ø‡ € ÙIòò
ò÷ð r   rU   )r‡   Úcollectionsr   Úurllib.errorr   Úurllib.parseÚurllib.requestÚ__all__Ú
namedtupler   r   r   r   r\   rU   © r   r   Ú<module>r¯      sd   ðñ
ó Û 	Û Û Û àÐ
€à×$Ò$ ]Ð4FÓG€òBò÷@.ñ @.÷DNñ N÷"(ó (r   