+
    ÖÇÜhâ  ã                   ór   € R t ^ RIHtHt ^ RIHtHtHt ^ RIH	t	 ^ RI
Ht  ! R R]4      t ! R R]4      tR	# )
z
Tokenizer Interface
)ÚABCÚabstractmethod)ÚIteratorÚListÚTuple)Ú
overridden)Ústring_span_tokenizec                   óv   a € ] tR t^t o Rt]V 3R lR l4       tV 3R lR ltV 3R lR ltV 3R lR	 lt	R
t
V tR# )Ú
TokenizerIzz
A processing interface for tokenizing a string.
Subclasses must define ``tokenize()`` or ``tokenize_sents()`` (or both).
c                ó6   <€ V ^8„  d   QhRS[ RS[S[ ,          /# ©é   ÚsÚreturn)Ústrr   )ÚformatÚ__classdict__s   "€Ú3/usr/lib/python3/dist-packages/nltk/tokenize/api.pyÚ__annotate__ÚTokenizerI.__annotate__   s   ø€ ÷ /ñ /™#ð /¡$¡s¥)ñ /ó    c                ón   € \        V P                  4      '       d   V P                  V.4      ^ ,          # R# )z4
Return a tokenized copy of *s*.

:rtype: List[str]
N)r   Útokenize_sents©Úselfr   s   &&r   ÚtokenizeÚTokenizerI.tokenize   s2   € ô �d×)Ñ)×*Ò*Ø×&Ñ&¨ sÓ+¨AÕ.Ð.ñ +r   c                óL   <€ V ^8„  d   QhRS[ RS[S[S[S[3,          ,          /# r   )r   r   r   Úint)r   r   s   "€r   r   r   $   s(   ø€ ÷ $ñ $™sð $¡x±±c¹3°hµÕ'@ñ $r   c                ó   € \        4       h)z›
Identify the tokens using integer offsets ``(start_i, end_i)``,
where ``s[start_i:end_i]`` is the corresponding token.

:rtype: Iterator[Tuple[int, int]]
©ÚNotImplementedErrorr   s   &&r   Úspan_tokenizeÚTokenizerI.span_tokenize$   s   € ô "Ó#Ð#r   c                óV   <€ V ^8„  d   QhRS[ S[,          RS[ S[ S[,          ,          /# ©r   Ústringsr   )r   r   )r   r   s   "€r   r   r   -   s'   ø€ ÷ 3ñ 3¡d©3¥ið 3±D¹¹c½µOñ 3r   c                óL   € V Uu. uF  q P                  V4      NK  	  up# u upi )z‹
Apply ``self.tokenize()`` to each element of ``strings``.  I.e.:

    return [self.tokenize(s) for s in strings]

:rtype: List[List[str]]
)r   ©r   r&   r   s   && r   r   ÚTokenizerI.tokenize_sents-   s#   € ñ +2Ó2©' Q—‘˜aÖ ©'Ñ2Ð2ùÒ2s   …!c                ól   <€ V ^8„  d   QhRS[ S[,          RS[S[ S[S[S[3,          ,          ,          /# r%   )r   r   r   r   r   )r   r   s   "€r   r   r   7   s3   ø€ ÷ .ñ .Ù™C•yð.á	‘$‘u™S¡#˜X•Õ'Õ	(ñ.r   c              #  óX   "  € V F  p\        V P                  V4      4      x € K!  	  R# 5i)z›
Apply ``self.span_tokenize()`` to each element of ``strings``.  I.e.:

    return [self.span_tokenize(s) for s in strings]

:yield: List[Tuple[int, int]]
N)Úlistr"   r(   s   && r   Úspan_tokenize_sentsÚTokenizerI.span_tokenize_sents7   s'   é € ó ˆAÜ�t×)Ñ)¨!Ó,Ó-Ô-ó ùs   ‚(*© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   r"   r   r-   Ú__static_attributes__Ú__classdictcell__©r   s   @r   r
   r
      s<   ø‡ € ñð
 ÷/ó ð/÷$ð $÷3ð 3÷.ö .r   r
   c                   óJ   a € ] tR t^Et o Rt]]R 4       4       tR tR t	Rt
V tR# )ÚStringTokenizerzpA tokenizer that divides a string into substrings by splitting
on the specified string (defined in subclasses).
c                ó   € \         h©Nr    )r   s   &r   Ú_stringÚStringTokenizer._stringJ   s
   € ô "Ð!r   c                ó8   € VP                  V P                  4      # r;   )Úsplitr<   r   s   &&r   r   ÚStringTokenizer.tokenizeO   s   € Ø�w‰w�t—|‘|Ó$Ð$r   c              #  óL   "  € \        WP                  4       R j  x€L
  R #  L5ir;   )r   r<   r   s   &&r   r"   ÚStringTokenizer.span_tokenizeR   s   é € Ü'¨¯<©<Ó8×8Ô8ùs   ‚$›"œ$r/   N)r0   r1   r2   r3   r4   Úpropertyr   r<   r   r"   r5   r6   r7   s   @r   r9   r9   E   s5   ø‡ € ñð Øñ"ó ó ð"ò%÷9ð 9r   r9   N)r4   Úabcr   r   Útypingr   r   r   Únltk.internalsr   Únltk.tokenize.utilr   r
   r9   r/   r   r   Ú<module>rH      s4   ðñ÷ $ß (Ñ (å %Ý 3ô..�ô ..ôb9�jö 9r   