+
    ÖÇÜhHy  ã                   óÒ   € R t ^ RIHtHt ^ RIt^ RIHtHtHtH	t	H
t
HtHtHt ^ RIHt R t]P"                  P%                  ]! 4       '       * RR7      t ! R R	4      t ! R
 R4      tR# )zC
Unit tests for nltk.tokenize.
See also nltk/test/tokenize.doctest
)ÚListÚTupleN)ÚLegalitySyllableTokenizerÚStanfordSegmenterÚSyllableTokenizerÚTreebankWordTokenizerÚTweetTokenizerÚpunktÚsent_tokenizeÚword_tokenize)ÚCharTokenizerc                  ó„   €  \        4       p V P                  R 4       V P                  R4       R#   \         d     R# i ; i)ÚarÚzhTF)r   Údefault_configÚLookupError)Úsegs    Ú>/usr/lib/python3/dist-packages/nltk/test/unit/test_tokenize.pyÚload_stanford_segmenterr      s@   € ðÜÓ!ˆØ×Ñ˜4Ô Ø×Ñ˜4Ô ÙøÜô Úðús   ‚,0 °?¾?z/NLTK was unable to find stanford-segmenter.jar.)Úreasonc                   ó¨  a € ] tR t^'t o R t]P                  P                  RR. R<O. R=O33R. R>O. R?O33R. R@O. R@O33R. RAO. RAO33R. RBO. RCO33R. RDO. REO33R	. RFO. RGO33R
. RHO. RIO33.4      V 3R lR l4       tR t	R t
R t]R 4       t]R 4       tR tR tR tR tR tR tR tR tR tR tR tR t]P                  P                  R. RJO4      RKR  l4       tR! t]P                  P                  R"R#. RLO3R$. RMO3R'R'.3R(R(.3R)R*R&.3R+R%R,.3R-. RNO3R.R/R&.3R0R1R&.3R2. ROO3R3. ROO3R4R5R6.3.4      V 3R7 lR8 l4       tV 3R9 lR: ltR;tV t R# )PÚTestTokenizec                ó^   € \        RRR7      pRpVP                  V4      p. ROpW48X  g   Q hR# )zG
Test TweetTokenizer using words with special and accented characters.
T)Ústrip_handlesÚ
reduce_lenuA   @myke: Let's test these words: resumÃ© EspaÃ±a MÃ¼nchen franÃ§aisN)
Ú:zLet'sÚtestÚtheseÚwordsr   u   resumÃ©u   EspaÃ±au   MÃ¼nchenu	   franÃ§ais©r   Útokenize)ÚselfÚ	tokenizerÚs9ÚtokensÚexpecteds   &    r   Útest_tweet_tokenizerÚ!TestTokenize.test_tweet_tokenizer(   s:   € ô
 #°À$ÔGˆ	ØPˆØ×#Ñ# BÓ'ˆò
ˆð Ô!Ð!Ò!ó    ztest_input, expectedsz#My text 0106404243030 is great textzMy ticket id is 1234543124123z<@remy: This is waaaaayyyy too much for you!!!!!! 01064042430z*My number is 06-46124080, except it's not.z+My number is 601-984-4813, except it's not.z/My number is (393)  928 -3010, except it's not.z1The product identification number is 48103284512.z(My favourite substraction is 240 - 1353.c                ó\   <€ V ^8„  d   QhRS[ RS[S[S[ ,          S[S[ ,          3,          /# )é   Ú
test_inputÚ	expecteds)Ústrr   r   )ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚTestTokenize.__annotate__ð   s/   ø€ ÷ )ñ )Ùð)Ù*/±±Sµ	¹4Á½9Ð0DÕ*Eñ)r(   c                ó‚   € \        RR.V4       F,  w  r4\        RRVR7      pVP                  V4      pWd8X  d   K,  Q h	  R# )aF  
Test `match_phone_numbers` in TweetTokenizer.

Note that TweetTokenizer is also passed the following for these tests:
    * strip_handles=True
    * reduce_len=True

:param test_input: The input string to tokenize using TweetTokenizer.
:type test_input: str
:param expecteds: A 2-tuple of tokenized sentences. The first of the two
    tokenized is the expected output of tokenization with `match_phone_numbers=True`.
    The second of the two tokenized lists is the expected output of tokenization
    with `match_phone_numbers=False`.
:type expecteds: Tuple[List[str], List[str]]
TF)r   r   Úmatch_phone_numbersN)Úzipr   r    )r!   r+   r,   r3   r%   r"   Ú	predicteds   &&&    r   Útest_tweet_tokenizer_expandedÚ*TestTokenize.test_tweet_tokenizer_expanded>   sP   € ôH .1°$¸°À	Ö-JÑ)ÐÜ&Ø"ØØ$7ôˆIð
 "×*Ñ*¨:Ó6ˆIØÖ(Ð(Ð(ó .Kr(   c                óR   € \        4       pVP                  R4      pV. RO8X  g   Q hR# )ú#
Test SyllableTokenizer tokenizer.
ÚjustificationN)ÚjusÚtiÚfiÚcaÚtion©r   r    )r!   r"   r$   s   &  r   Ú+test_sonority_sequencing_syllable_tokenizerÚ8TestTokenize.test_sonority_sequencing_syllable_tokenizer  s+   € ô &Ó'ˆ	Ø×#Ñ# OÓ4ˆØÒ:Ô:Ð:Ò:r(   c                ó`   € \        4       pRR,          pVP                  V4      pW2.8X  g   Q hR# )r9   Ú9i'  Nr@   )r!   r"   Útextr$   s   &   r   Útest_syllable_tokenizer_numbersÚ,TestTokenize.test_syllable_tokenizer_numbers  s3   € ô &Ó'ˆ	Ø�U�{ˆØ×#Ñ# DÓ)ˆØ˜ÔÐÒr(   c                ó€   € ^ RI Hp Rp\        VP                  4       4      pVP                  V4      pV. RO8X  g   Q hR# )z+
Test LegalitySyllableTokenizer tokenizer.
)r   Ú	wonderfulN)ÚwonÚderÚful)Únltk.corpusr   r   r    )r!   r   Ú	test_wordr"   r$   s   &    r   Ú*test_legality_principle_syllable_tokenizerÚ7TestTokenize.test_legality_principle_syllable_tokenizer  s;   € õ 	&àˆ	Ü-¨e¯k©k«mÓ<ˆ	Ø×#Ñ# IÓ.ˆØÒ.Ô.Ð.Ò.r(   c                ó°   € \        4       pVP                  R4       RpVP                  VP                  4       4      pVP                  4       . RO8X  g   Q hR# )z>
Test the Stanford Word Segmenter for Arabic (default config)
r   un   ÙŠØ¨Ø­Ø« Ø¹Ù„Ù… Ø§Ù„Ø­Ø§Ø³ÙˆØ¨ Ø§Ø³ØªØ®Ø¯Ø§Ù… Ø§Ù„Ø­ÙˆØ³Ø¨Ø© Ø¨Ø¬Ù…ÙŠØ¹ Ø§Ø´ÙƒØ§Ù„Ù‡Ø§ Ù„Ø­Ù„ Ø§Ù„Ù…Ø´ÙƒÙ„Ø§ØªN)u   ÙŠØ¨Ø­Ø«u   Ø¹Ù„Ù…u   Ø§Ù„Ø­Ø§Ø³ÙˆØ¨u   Ø§Ø³ØªØ®Ø¯Ø§Ù…u   Ø§Ù„Ø­ÙˆØ³Ø¨Ø©u   Ø¨u   Ø¬Ù…ÙŠØ¹u
   Ø§Ø´ÙƒØ§Ù„u   Ù‡Ø§u   Ù„u   Ø­Ù„u   Ø§Ù„Ù…Ø´ÙƒÙ„Ø§Øª©r   r   ÚsegmentÚsplit©r!   r   ÚsentÚsegmented_sents   &   r   Útest_stanford_segmenter_arabicÚ+TestTokenize.test_stanford_segmenter_arabic'  sS   € ô
  Ó!ˆØ×Ñ˜4Ô ØˆØŸ™ T§Z¡Z£\Ó2ˆØ×#Ñ#Ó%ò *
ô 
ð 	
ò 
r(   c                ó°   € \        4       pVP                  R4       RpVP                  VP                  4       4      pVP                  4       . RO8X  g   Q hR# )z?
Test the Stanford Word Segmenter for Chinese (default config)
r   u$   è¿™æ˜¯æ–¯å�¦ç¦�ä¸­æ–‡åˆ†è¯�å™¨æµ‹è¯•N)u   è¿™u   æ˜¯u	   æ–¯å�¦ç¦�u   ä¸­æ–‡u	   åˆ†è¯�å™¨u   æµ‹è¯•rR   rU   s   &   r   Útest_stanford_segmenter_chineseÚ,TestTokenize.test_stanford_segmenter_chinese?  sS   € ô
  Ó!ˆØ×Ñ˜4Ô Ø5ˆØŸ™ T§Z¡Z£\Ó2ˆØ×#Ñ#Ó%ò *
ô 
ð 	
ò 
r(   c                ó”   € \        4       pRpR.pVP                  V4      pWC8X  g   Q hRp. ROpVP                  V4      pWC8X  g   Q hR# )zD
Test a string that resembles a phone number but contains a newline
ú(393)  928 -3010z(393)
928 -3010N)Ú(Ú393Ú)z	928 -3010r   )r!   r"   Útest1r%   ÚresultÚtest2s   &     r   Útest_phone_tokenizerÚ!TestTokenize.test_phone_tokenizerQ  s]   € ô #Ó$ˆ	Ø"ˆØ&Ð'ˆØ×#Ñ# EÓ*ˆØÔ!Ð!Ð!ð #ˆÚ1ˆØ×#Ñ# EÓ*ˆØÔ!Ð!Ò!r(   c                óŠ  € \        4       pRpR.pVP                  V4      pWC8X  g   Q hRpR.pVP                  V4      pWC8X  g   Q hRp. ROpVP                  V4      pWC8X  g   Q hRp. R	OpVP                  V4      pWC8X  g   Q hRp. R
OpVP                  V4      pWC8X  g   Q hRp	. ROpVP                  V	4      pWC8X  g   Q hR# )zH
Test a string that contains Emoji ZWJ Sequences and skin tone modifier
u   ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘§â€�ðŸ‘§u   ðŸ‘¨ðŸ�¿uŒ   ðŸ¤” ðŸ™ˆ me asÃ­, seðŸ˜Œ ds ðŸ’•ðŸ‘­ðŸ‘™ hello ðŸ‘©ðŸ�¾â€�ðŸŽ“ emoji hello ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘¦â€�ðŸ‘¦ how are ðŸ˜Š you todayðŸ™…ðŸ�½ðŸ™…ðŸ�½u   ðŸ‡¦ðŸ‡µðŸ‡µðŸ‡±ðŸ‡ªu   Hi ðŸ‡¨ðŸ‡¦, ðŸ˜�!!u   <3 ðŸ‡¨ðŸ‡¦ ðŸ¤� ðŸ‡µðŸ‡± <3N)u   ðŸ¤”u   ðŸ™ˆÚmeu   asÃ­Ú,Úseu   ðŸ˜ŒÚdsu   ðŸ’•u   ðŸ‘­u   ðŸ‘™Úhellou   ðŸ‘©ðŸ�¾â€�ðŸŽ“Úemojirl   u   ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘¦â€�ðŸ‘¦ÚhowÚareu   ðŸ˜ŠÚyouÚtodayõ   ðŸ™…ðŸ�½rr   )u   ðŸ‡¦ðŸ‡µõ   ðŸ‡µðŸ‡±u   ðŸ‡ª)ÚHiõ   ðŸ‡¨ðŸ‡¦ri   u   ðŸ˜�Ú!rv   )ú<3ru   u   ðŸ¤�rs   rw   r   )
r!   r"   rb   r%   rc   rd   Útest3Útest4Útest5Útest6s
   &         r   Útest_emoji_tokenizerÚ!TestTokenize.test_emoji_tokenizerd  s  € ô #Ó$ˆ	ð ,ˆØ/Ð0ˆØ×#Ñ# EÓ*ˆØÔ!Ð!Ð!ð ˆØ�<ˆØ×#Ñ# EÓ*ˆØÔ!Ð!Ð!ð _ˆò
ˆð2 ×#Ñ# EÓ*ˆØÔ!Ð!Ð!ð 'ˆÚ3ˆØ×#Ñ# EÓ*ˆØÔ!Ð!Ð!à%ˆÚ<ˆØ×#Ñ# EÓ*ˆØÔ!Ð!Ð!à.ˆÚ?ˆØ×#Ñ# EÓ*ˆØÔ!Ð!Ò!r(   c                ó6   € Rp. ROp\        V4      V8X  g   Q hR# )z1
Test padding of asterisk for word tokenization.
z1This is a, *weird sentence with *asterisks in it.N)ÚThisÚisÚari   Ú*ÚweirdÚsentenceÚwithr‚   Ú	asterisksÚinÚitÚ.©r   ©r!   rE   r%   s   &  r   Útest_pad_asteriskÚTestTokenize.test_pad_asterisk¥  s&   € ð Cˆò
ˆô ˜TÓ" hÔ.Ð.Ò.r(   c                ó6   € Rp. ROp\        V4      V8X  g   Q hR# )z0
Test padding of dotdot* for word tokenization.
zPWhy did dotdot.. not get tokenized but dotdotdot... did? How about manydots.....N)ÚWhyÚdidÚdotdotz..ÚnotÚgetÚ	tokenizedÚbutÚ	dotdotdotz...r�   Ú?ÚHowÚaboutÚmanydotsz.....rŠ   r‹   s   &  r   Útest_pad_dotdotÚTestTokenize.test_pad_dotdot»  s&   € ð bˆò
ˆô$ ˜TÓ" hÔ.Ð.Ò.r(   c                ód  € \        RR7      pRp. ROpVP                  V4      pWC8X  g   Q hRp. R OpVP                  V4      pWC8X  g   Q hRp. RNRNRNRNR	NRNR
NRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNpVP                  V4      pWC8X  g   Q hRp. R!OpVP                  V4      pWC8X  g   Q hRp. R"OpVP                  V4      pWC8X  g   Q hRp	. R#OpVP                  V	4      pWC8X  g   Q hRp
. R$OpVP                  V
4      pWC8X  g   Q hR# )%zG
Test remove_handle() from casual.py with specially crafted edge cases
T)r   z-@twitter hello @twi_tter_. hi @12345 @123newsu]   @n`@n~@n(@n)@n-@n=@n+@n\@n|@n[@n]@n{@n}@n;@n:@n'@n"@n/@n?@n.@n,@n<@n>@n @n
@n Ã±@n.Ã¼@n.Ã§@n.zKa@n j@n z@n A@n L@n Z@n 1@n 4@n 7@n 9@n 0@n _@n !@n @@n #@n $@n %@n &@n *@nr�   ú@nÚjÚzÚAÚLÚZÚ1Ú4Ú7rD   Ú0Ú_rv   Ú@Ú#Ú$Ú%Ú&r‚   z@n!a @n#a @n$a @n%a @n&a @n*azD@n!@n @n#@n @n$@n @n%@n @n&@n @n*@n @n@n @@n @n@@n @n_@n @n7@n @nj@nz^@abcdefghijklmnopqrstuvwxyz @abcdefghijklmno1234 @abcdefghijklmno_ @abcdefghijklmnoendofhandlez^@abcdefghijklmnop@abcde @abcdefghijklmno@abcde @abcdefghijklmno_@abcde @abcdefghijklmno5@abcdeN)rl   r‰   Úhi)Ú`Ú~r_   ra   Ú-Ú=Ú+Ú\Ú|Ú[Ú]Ú{Ú}Ú;r   Ú'Ú"Ú/r—   r‰   ri   Ú<Ú>ô   Ã±r‰   ô   Ã¼r‰   ô   Ã§r‰   )rv   r�   rª   r�   r«   r�   r¬   r�   r­   r�   r‚   r�   )rv   rž   rª   rž   r«   rž   r¬   rž   r­   rž   r‚   rž   rž   rž   r©   rž   rž   r©   rž   z@n_rž   z@n7rž   z@njrž   )ÚpqrstuvwxyzÚ1234r¨   Úendofhandle)Úpú@abcdez@abcdefghijklmnorÇ   r¨   rÇ   Ú5rÇ   r   )r!   r"   rb   r%   rc   rd   rx   ry   rz   r{   Útest7s   &          r   Útest_remove_handleÚTestTokenize.test_remove_handleÔ  sÃ  € ô
 #°Ô6ˆ	ð @ˆÚ'ˆØ×#Ñ# EÓ*ˆØÔ!Ð!Ð!ð sˆò
ˆð> ×#Ñ# EÓ*ˆØÔ!Ð!Ð!ð ^ˆð'
Øð'
àð'
ð ð'
ð ð	'
ð
 ð'
ð ð'
ð ð'
ð ð'
ð ð'
ð ð'
ð ð'
ð ð'
ð ð'
ð ð'
ð ð'
ð  ð!'
ð" ð#'
ð$ ð%'
ð& ð''
ð( ð)'
ð* ð+'
ð, ð-'
ð. ð/'
ð0 ð1'
ð2 ð3'
ð4 ð5'
ð6 ð7'
ð8 ð9'
ð: ð;'
ð< ð='
ð> ð?'
ð@ ðA'
ðB ðC'
ðD ðE'
ðF ðG'
ðH ðI'
ðJ ðK'
ðL ðM'
ˆðP ×#Ñ# EÓ*ˆØÔ!Ð!Ð!ð 0ˆÚOˆØ×#Ñ# EÓ*ˆØÔ!Ð!Ð!ð Wˆò
ˆð6 ×#Ñ# EÓ*ˆØÔ!Ð!Ð!ð qˆÚ>ˆØ×#Ñ# EÓ*ˆØÔ!Ð!Ð!ð qˆò	
ˆð ×#Ñ# EÓ*ˆØÔ!Ð!Ò!r(   c                ó
  € \        4       pRp. ROp\        VP                  V4      4      pWC8X  g   Q hRp. ROp\        VP                  V4      4      pWC8X  g   Q hRp. ROp\        VP                  V4      4      pWC8X  g   Q hR# )z3
Test TreebankWordTokenizer.span_tokenize function
zNGood muffins cost $3.88
in New (York).  Please (buy) me
two of them.
(Thanks).zmThe DUP is similar to the "religious right" in the United States and takes a hardline stance on social issueszqThe DUP is similar to the "religious right" in the United States and takes a ``hardline'' stance on social issuesN))é    é   )é   é   )é   é   )é   é   )rÔ   é   )é   é   )é   é   )é   é    )rÛ   é$   )rÜ   é%   )rÝ   é&   )é(   é.   )é/   é0   )râ   é3   )rã   é4   )é5   é7   )é8   é;   )é<   é>   )é?   éD   )éE   éF   )rî   éL   )rï   éM   )rð   éN   )©rÍ   é   ©rÎ   é   ©é   é
   ©é   rÓ   ©rÔ   é   ©é   é   ©r×   rØ   ©rØ   rÜ   ©rÝ   é*   ©r  é+   ©é,   rà   ©rá   é2   ©rã   é9   ©é:   é@   ©éA   rì   ©rí   éJ   ©éK   rï   )rð   éU   )éV   é\   )é]   é_   )é`   éf   )ég   ém   )rò   rô   rö   rù   rû   rý   r   r  r  r  r  r  r
  r  r  r  r  )rð   éO   )r  éW   )r  éY   )éZ   r  )éa   éc   )éd   éj   )ék   éq   )r   ÚlistÚspan_tokenize)r!   r"   rb   r%   rc   rd   rx   s   &      r   Útest_treebank_span_tokenizerÚ)TestTokenize.test_treebank_span_tokenizerm  s¢   € ô
 *Ó+ˆ	ð dˆò
ˆô2 �i×-Ñ-¨eÓ4Ó5ˆØÔ!Ð!Ð!ð @ˆò
ˆô0 �i×-Ñ-¨eÓ4Ó5ˆØÔ!Ð!Ð!ð Fˆò
ˆô4 �i×-Ñ-¨eÓ4Ó5ˆØÔ!Ð!Ò!r(   c                óf   € Rp. ROp\        V4      V8X  g   Q hRp. ROp\        V4      V8X  g   Q hR# )z
Test word_tokenize function
z0The 'v', I've been fooled but I'll seek revenge.z'v' 're'N)ÚTher»   Úvr»   ri   ÚIz'veÚbeenÚfooledr•   r/  z'llÚseekÚrevenger‰   )r»   r.  r»   z'rer»   rŠ   )r!   r„   r%   s   &  r   Útest_word_tokenizeÚTestTokenize.test_word_tokenizeÎ  sC   € ð
 Fˆò
ˆô" ˜XÓ&¨(Ô2Ð2Ð2àˆÚ.ˆÜ˜XÓ&¨(Ô2Ð2Ò2r(   c                ó    € R RR.3R. RO3R. RO3.pV F1  w  r#\         P                  ! V4       Uu. uF  qDNK  	  ppWS8X  d   K1  Q h	  R# u upi )Ú12NÚ123rÄ   ©r¤   Ú2)r:  N)r9  ©r:  Ú3)r<  N)r9  r;  )r<  r¥   )r¥   N)r	   Ú
_pair_iter)r!   Ú
test_casesr+   Úexpected_outputÚxÚactual_outputs   &     r   Útest_punkt_pair_iterÚ!TestTokenize.test_punkt_pair_iterë  sg   € à�J Ð,Ð-ØÒ9Ð:ØÒFÐGð
ˆ
ó ,6Ñ'ˆJÜ(-×(8Ò(8¸Ô(DÓEÑ(D 1šQÑ(DˆMÐEà Ö3Ð3Ð3ó ,6ùÚEs   ±
Ac                ó^   € \        . 4      p\        P                  ! V4      p\        V4       R # ©N)Úiterr	   r=  r(  )r!   rˆ   Úgens   &  r   Ú5test_punkt_pair_iter_handles_stop_iteration_exceptionÚBTestTokenize.test_punkt_pair_iter_handles_stop_iteration_exception÷  s"   € ä�"‹Xˆä×Ò˜rÓ"ˆäˆSŽ	r(   c                ó�   € \         P                  ! 4       p ! R  R4      pV! 4       Vn        \        VP	                  R4      4       R# )c                   ó&   a € ] tR tRt o R tRtV tR# )ÚkTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exception.<locals>.TestPunktTokenizeWordsMocki  c                ó   € \        . 4      # rE  )rF  )r!   Úss   &&r   r   ÚyTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exception.<locals>.TestPunktTokenizeWordsMock.word_tokenize  s   € Ü˜B“x�r(   © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Ú__static_attributes__Ú__classdictcell__©r/   s   @r   ÚTestPunktTokenizeWordsMockrL    s   ø‡ € ÷ ð  r(   rX  r   N)r	   ÚPunktBaseClassÚ
_lang_varsr(  Ú_tokenize_words)r!   ÚobjrX  s   &  r   Ú:test_punkt_tokenize_words_handles_stop_iteration_exceptionÚGTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exceptionÿ  s9   € Ü×"Ò"Ó$ˆ÷	 ñ 	 ñ 4Ó5ˆŒäˆS× Ñ  Ó(Ö)r(   c                ó¬   €  ! R  R\         P                  4      p\         P                  ! V! 4       R7      pRp. ROpVP                  V4      V8X  g   Q hR# )c                   ó   € ] tR tRtRtRtR# )ÚNTestTokenize.test_punkt_tokenize_custom_lang_vars.<locals>.BengaliLanguageVarsi  rP  N)r‰   r—   rv   u   à¥¤©rQ  rR  rS  rT  Úsent_end_charsrU  rP  r(   r   ÚBengaliLanguageVarsra    s   † Ø6ŒNr(   rd  )Ú	lang_varsõc  à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¶à§�à¦°à§€ à¦�à¦® à¦­à§‡à¦™à§�à¦•à¦¾à¦‡à§Ÿà¦¾ à¦¨à¦¾à¦‡à¦¡à§� à¦¸à§‹à¦®à¦¬à¦¾à¦° à¦†à¦‡ à¦†à¦‡ à¦Ÿà¦¿ à¦¦à¦¿à¦²à§�à¦²à¦¿à¦° à¦¹à§€à¦°à¦• à¦œà§Ÿà¦¨à§�à¦¤à§€ à¦‰à¦¦à¦¯à¦¾à¦ªà¦¨à§‡à¦° à¦‰à¦¦à§�à¦¬à§‹à¦§à¦¨ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤ à¦…à¦¨à¦²à¦¾à¦‡à¦¨à§‡à¦° à¦®à¦¾à¦§à§�à¦¯à¦®à§‡ à¦�à¦‡ à¦…à¦¨à§�à¦·à§�à¦ à¦¾à¦¨à§‡ à¦•à§‡à¦¨à§�à¦¦à§�à¦°à§€à§Ÿ à¦®à¦¾à¦¨à¦¬ à¦¸à¦®à§�à¦ªà¦¦ à¦‰à¦¨à§�à¦¨à§Ÿà¦¨à¦®à¦¨à§�à¦¤à§�à¦°à§€ à¦¶à§�à¦°à§€ à¦°à¦®à§‡à¦¶ à¦ªà§‹à¦–à¦°à¦¿à§Ÿà¦¾à¦² â€˜à¦¨à¦¿à¦¶à¦¾à¦™à§�à¦•â€™  à¦‰à¦ªà¦¸à§�à¦¥à¦¿à¦¤ à¦›à¦¿à¦²à§‡à¦¨à¥¤ à¦�à¦‡ à¦‰à¦ªà¦²à¦•à§�à¦·à§�à¦¯à§‡ à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¹à§€à¦°à¦•à¦œà§Ÿà¦¨à§�à¦¤à§€à¦° à¦²à§‹à¦—à§‹ à¦�à¦¬à¦‚ à§¨à§¦à§©à§¦-à¦�à¦° à¦œà¦¨à§�à¦¯ à¦ªà§�à¦°à¦¤à¦¿à¦·à§�à¦ à¦¾à¦¨à§‡à¦° à¦²à¦•à§�à¦·à§�à¦¯ à¦“ à¦ªà¦°à¦¿à¦•à¦²à§�à¦ªà¦¨à¦¾à¦° à¦¨à¦¥à¦¿ à¦ªà§�à¦°à¦•à¦¾à¦¶ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤N)u  à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¶à§�à¦°à§€ à¦�à¦® à¦­à§‡à¦™à§�à¦•à¦¾à¦‡à§Ÿà¦¾ à¦¨à¦¾à¦‡à¦¡à§� à¦¸à§‹à¦®à¦¬à¦¾à¦° à¦†à¦‡ à¦†à¦‡ à¦Ÿà¦¿ à¦¦à¦¿à¦²à§�à¦²à¦¿à¦° à¦¹à§€à¦°à¦• à¦œà§Ÿà¦¨à§�à¦¤à§€ à¦‰à¦¦à¦¯à¦¾à¦ªà¦¨à§‡à¦° à¦‰à¦¦à§�à¦¬à§‹à¦§à¦¨ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤u+  à¦…à¦¨à¦²à¦¾à¦‡à¦¨à§‡à¦° à¦®à¦¾à¦§à§�à¦¯à¦®à§‡ à¦�à¦‡ à¦…à¦¨à§�à¦·à§�à¦ à¦¾à¦¨à§‡ à¦•à§‡à¦¨à§�à¦¦à§�à¦°à§€à§Ÿ à¦®à¦¾à¦¨à¦¬ à¦¸à¦®à§�à¦ªà¦¦ à¦‰à¦¨à§�à¦¨à§Ÿà¦¨à¦®à¦¨à§�à¦¤à§�à¦°à§€ à¦¶à§�à¦°à§€ à¦°à¦®à§‡à¦¶ à¦ªà§‹à¦–à¦°à¦¿à§Ÿà¦¾à¦² â€˜à¦¨à¦¿à¦¶à¦¾à¦™à§�à¦•â€™  à¦‰à¦ªà¦¸à§�à¦¥à¦¿à¦¤ à¦›à¦¿à¦²à§‡à¦¨à¥¤u/  à¦�à¦‡ à¦‰à¦ªà¦²à¦•à§�à¦·à§�à¦¯à§‡ à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¹à§€à¦°à¦•à¦œà§Ÿà¦¨à§�à¦¤à§€à¦° à¦²à§‹à¦—à§‹ à¦�à¦¬à¦‚ à§¨à§¦à§©à§¦-à¦�à¦° à¦œà¦¨à§�à¦¯ à¦ªà§�à¦°à¦¤à¦¿à¦·à§�à¦ à¦¾à¦¨à§‡à¦° à¦²à¦•à§�à¦·à§�à¦¯ à¦“ à¦ªà¦°à¦¿à¦•à¦²à§�à¦ªà¦¨à¦¾à¦° à¦¨à¦¥à¦¿ à¦ªà§�à¦°à¦•à¦¾à¦¶ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤)r	   ÚPunktLanguageVarsÚPunktSentenceTokenizerr    )r!   rd  r\  Ú	sentencesr%   s   &    r   Ú$test_punkt_tokenize_custom_lang_varsÚ1TestTokenize.test_punkt_tokenize_custom_lang_vars
  sR   € ô	7¤%×"9Ñ"9ô 	7ô ×*Ò*Ñ5HÓ5JÔKˆð zˆ	ò
ˆð �|‰|˜IÓ&¨(Ô2Ð2Ò2r(   c                ój   € \         P                  ! 4       pR pR .pVP                  V4      V8X  g   Q hR# )rf  N)r	   rh  r    )r!   r\  ri  r%   s   &   r   Ú'test_punkt_tokenize_no_custom_lang_varsÚ4TestTokenize.test_punkt_tokenize_no_custom_lang_vars  s>   € Ü×*Ò*Ó,ˆð zˆ	ð rð
ˆð �|‰|˜IÓ&¨(Ô2Ð2Ò2r(   z%input_text,n_sents,n_splits,lang_varsNc                óà   € \         P                  ! 4       pVR 8w  d   WEn        \        VP	                  V4      4      V8X  g   Q h\        \        VP                  V4      4      4      V8X  g   Q hR # rE  )r	   rh  rZ  Úlenr    r(  Údebug_decisions)r!   Ú
input_textÚn_sentsÚn_splitsre  r"   s   &&&&& r   Úpunkt_debug_decisionsÚ"TestTokenize.punkt_debug_decisions&  sa   € ô$ ×0Ò0Ó2ˆ	Ø˜ÔØ#,Ô ä�9×%Ñ% jÓ1Ó2°gÔ=Ð=Ð=Ü”4˜	×1Ñ1°*Ó=Ó>Ó?À8ÔKÐKÒKr(   c                ól   €  ! R  R\         P                  4      pV P                  R^^V! 4       R7       R# )c                   ó   € ] tR tRtRtRtR# )ÚGTestTokenize.test_punkt_debug_decisions_custom_end.<locals>.ExtLangVarsiB  rP  N)r‰   r—   rv   Ú^rb  rP  r(   r   ÚExtLangVarsry  B  s   † Ø1ŒNr(   r{  z4Subject: Some subject^ Attachments: Some attachments)rs  rt  re  N)r	   rg  ru  )r!   r{  s   & r   Ú%test_punkt_debug_decisions_custom_endÚ2TestTokenize.test_punkt_debug_decisions_custom_end?  s7   € ô	2œ%×1Ñ1ô 	2ð 	×"Ñ"ØBØØÙ!“mð	 	#ö 	
r(   zsentences, expectedzthis is a test. . new sentence.zThis. . . ThatúThis.ÚThatzThis..... ThatzThis... ThatzThis.. . ThatzThis.. .zThis. .. Thatz.. ThatzThis. ,. ThatzThis!!! ThatzThis!!!z
This! ThatzThis!z+1. This is R .
2. This is A .
3. That's allz+1. This is R .	2. This is A .	3. That's allzHello.	TherezHello.ÚTherec                ó6   <€ V ^8„  d   QhRS[ RS[S[ ,          /# )r*   ri  r%   )r-   r   )r.   r/   s   "€r   r0   r1   h  s   ø€ ÷ 4ñ 4©Cð 4¹4Á½9ñ 4r(   c                ó*   € \        V4      V8X  g   Q hR # rE  )r
   )r!   ri  r%   s   &&&r   Útest_sent_tokenizeÚTestTokenize.test_sent_tokenizeN  s   € ô6 ˜YÓ'¨8Ô3Ð3Ò3r(   c                ó   <€ V ^8„  d   QhRR/# ©r*   ÚreturnNrP  )r.   r/   s   "€r   r0   r1   k  s   ø€ ÷ 
ñ 
 tñ 
r(   c                ó¦   € R p\        4       pVP                  V4      \        V4      8X  g   Q h\        VP                  V4      4      . RO8X  g   Q hR# )zHello thereN))rÍ   é   )r‰  r*   )r*   ró   )ró   rÎ   )rÎ   rÏ   )rÏ   é   )rŠ  rõ   )rõ   r÷   )r÷   é	   )r‹  rø   )rø   rú   )r   r    r(  r)  )r!   r„   r"   s   &  r   Útest_string_tokenizerÚ"TestTokenize.test_string_tokenizerk  sR   € Ø ˆÜ!“Oˆ	Ø×!Ñ! (Ó+¬t°H«~Ô=Ð=Ð=Ü�I×+Ñ+¨HÓ5Ó6ò ;
ô 
ð 	
ò 
r(   rP  )ÚMyrE   Ú01064042430Ú30r€   ÚgreatrE   )rŽ  rE   Ú0106404243030r€   r‘  rE   )rŽ  ÚticketÚidr€   Ú12345431241Ú23)rŽ  r“  r”  r€   Ú1234543124123)r   r   r€   ÚwaaayyyÚtooÚmuchÚforrp   rv   rv   rv   r�  )	rŽ  Únumberr€   z06-46124080ri   Úexceptúit'sr’   r‰   )	rŽ  rœ  r€   z601-984-4813ri   r�  rž  r’   r‰   )
rŽ  rœ  r€   z601-984-Ú4813ri   r�  rž  r’   r‰   )	rŽ  rœ  r€   r^   ri   r�  rž  r’   r‰   )rŽ  rœ  r€   r_   r`   ra   Ú928r±   Ú3010ri   r�  rž  r’   r‰   )r-  ÚproductÚidentificationrœ  r€   Ú
4810328451r:  r‰   )r-  r¢  r£  rœ  r€   Ú48103284512r‰   )rŽ  Ú	favouriteÚsubstractionr€   z
240 - 1353r‰   )rŽ  r¦  r§  r€   Ú240r±   Ú1353r‰   ))z4Subject: Some subject. Attachments: Some attachmentsr*   r‰  )z4Subject: Some subject! Attachments: Some attachmentsr*   r‰  )z4This is just a normal sentence, just like any other.r‰  rÍ   rE  )zthis is a test.r‰   znew sentence.)r~  r‰   r‰   r  )r~  z,.r  )z1.zThis is R .z2.zThis is A .z3.z
That's all)!rQ  rR  rS  rT  r&   ÚpytestÚmarkÚparametrizer6   rA   rF   rO   Úcheck_stanford_segmenterrX   r[   re   r|   rŒ   r›   rÊ   r*  r4  rB  rH  r]  rj  rm  ru  r|  rƒ  rŒ  rU  rV  rW  s   @r   r   r   '   sp  ø‡ € ò"ð, ‡[�[×ÑØð 6âNÚJððð 0âEÚAððð Oòòðð ðL =ò
ò
ððð< >ò
òððð> Bò
òðð ðF Dò	òððð6 ;âPÚVðððMn	
óq÷d)óeqðd)ò6;ò ò	/ð ñ
ó ð
ð. ñ
ó ð
ò""ò&?"òB/ò,/ò2W"òr_"òB3ò:
4òò	*ò3ò"	3ð ‡[�[×ÑØ/ò	
óó"Ló#ð"Lò
ð ‡[�[×ÑØð 2Ú9ðð Ò:Ð;ØÐ 0Ð1Ð2Ø˜nÐ-Ð.Ø˜z¨6Ð2Ð3Ø˜w¨	Ð2Ð3ØÒ5Ð6Ø˜i¨Ð0Ð1Ø˜G VÐ,Ð-à?ÚNðð
 @ÚNðð ˜x¨Ð1Ð2ð+	
ó÷44ó5ð44÷
ö 
r(   r   c                   óV   a € ] tR tRt o V 3R lR ltV 3R lR ltV 3R lR ltRtV tR	# )
ÚTestPunktTraineri~  c                ó   <€ V ^8„  d   QhRR/# r†  rP  )r.   r/   s   "€r   r0   ÚTestPunktTrainer.__annotate__  s   ø€ ÷ )ñ ) $ñ )r(   c                óR   € \         P                  ! 4       pVP                  R 4       R# )zThis is a test.N©r	   ÚPunktTrainerÚtrain©r!   Útrainers   & r   Útest_punkt_trainÚ!TestPunktTrainer.test_punkt_train  s   € Ü×$Ò$Ó&ˆØ�‰Ð'Ö(r(   c                ó   <€ V ^8„  d   QhRR/# r†  rP  )r.   r/   s   "€r   r0   r±  ƒ  s   ø€ ÷ ñ ¨dñ r(   c                óR   € \         P                  ! 4       pVP                  R 4       R# )r~  Nr³  r¶  s   & r   Útest_punkt_train_single_wordÚ-TestPunktTrainer.test_punkt_train_single_wordƒ  s   € Ü×$Ò$Ó&ˆØ�‰�gÖr(   c                ó   <€ V ^8„  d   QhRR/# r†  rP  )r.   r/   s   "€r   r0   r±  ‡  s   ø€ ÷ (ñ (¨$ñ (r(   c                óR   € \         P                  ! 4       pVP                  R 4       R# )zThis is a testNr³  r¶  s   & r   Útest_punkt_train_no_puncÚ)TestPunktTrainer.test_punkt_train_no_punc‡  s   € Ü×$Ò$Ó&ˆØ�‰Ð&Ö'r(   rP  N)	rQ  rR  rS  rT  r¸  r¼  rÀ  rU  rV  rW  s   @r   r¯  r¯  ~  s#   ø‡ € ÷)ð )÷ð ÷(ö (r(   r¯  )Ú__doc__Útypingr   r   rª  Únltk.tokenizer   r   r   r   r   r	   r
   r   Únltk.tokenize.simpler   r   r«  Úskipifr­  r   r¯  rP  r(   r   Ú<module>rÇ     sj   ðñ÷
 ã ÷	÷ 	ó 	õ /òð "Ÿ;™;×-Ñ-ÙÓ!Ô!Ø<ð .ó Ð ÷T
ñ T
÷n(ó (r(   