+
    ÖÇÜhu  ã                   óT   € R t ^ RIt^ RIt^ RIHt ^ RIHt ^ RIHt  ! R R]4      t	R# )a  
This is a NLTK port of the tokenizer used in the NIST BLEU evaluation script,
https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/mteval-v14.pl#L926
which was also ported into Python in
https://github.com/lium-lst/nmtpy/blob/master/nmtpy/metrics/mtevalbleu.py#L162
N)Úperluniprops)Ú
TokenizerI)Úxml_unescapec            	       ó�  a € ] tR t^t o Rt]P                  ! R4      R3t]P                  ! R4      R3t]P                  ! R4      R3t	]P                  ! R4      R	3t
]P                  ! R
4      R3t]P                  ! R4      R	3t]	]
]].t]! RP                  ]! ]P$                  ! R4      4      4      4      t]! RP                  ]! ]P$                  ! R4      4      4      4      t]! RP                  ]! ]P$                  ! R4      4      4      4      t]P,                  ! RR]4      t]P,                  ! RR]4      t]P,                  ! RR]4      t]P                  ! R4      R3t]P                  ! R] R] R24      R	3t]P                  ! R] R] R24      R3t]P                  ! R] R24      R3t]]]].tR tRR lt RR lt!Rt"V t#R# )ÚNISTTokenizeruÄ  
This NIST tokenizer is sentence-based instead of the original
paragraph-based tokenization from mteval-14.pl; The sentence-based
tokenization is consistent with the other tokenizers available in NLTK.

>>> from nltk.tokenize.nist import NISTTokenizer
>>> nist = NISTTokenizer()
>>> s = "Good muffins cost $3.88 in New York."
>>> expected_lower = [u'good', u'muffins', u'cost', u'$', u'3.88', u'in', u'new', u'york', u'.']
>>> expected_cased = [u'Good', u'muffins', u'cost', u'$', u'3.88', u'in', u'New', u'York', u'.']
>>> nist.tokenize(s, lowercase=False) == expected_cased
True
>>> nist.tokenize(s, lowercase=True) == expected_lower  # Lowercased.
True

The international_tokenize() is the preferred function when tokenizing
non-european text, e.g.

>>> from nltk.tokenize.nist import NISTTokenizer
>>> nist = NISTTokenizer()

# Input strings.
>>> albb = u'Alibaba Group Holding Limited (Chinese: é˜¿é‡Œå·´å·´é›†å›¢æŽ§è‚¡ æœ‰é™�å…¬å�¸) is a Chinese e-commerce company...'
>>> amz = u'Amazon.com, Inc. (/ËˆÃ¦mÉ™zÉ’n/) is an American electronic commerce...'
>>> rkt = u'Rakuten, Inc. (æ¥½å¤©æ ªå¼�ä¼šç¤¾ Rakuten Kabushiki-gaisha) is a Japanese electronic commerce and Internet company based in Tokyo.'

# Expected tokens.
>>> expected_albb = [u'Alibaba', u'Group', u'Holding', u'Limited', u'(', u'Chinese', u':', u'é˜¿é‡Œå·´å·´é›†å›¢æŽ§è‚¡', u'æœ‰é™�å…¬å�¸', u')']
>>> expected_amz = [u'Amazon', u'.', u'com', u',', u'Inc', u'.', u'(', u'/', u'ËˆÃ¦', u'm']
>>> expected_rkt = [u'Rakuten', u',', u'Inc', u'.', u'(', u'æ¥½å¤©æ ªå¼�ä¼šç¤¾', u'Rakuten', u'Kabushiki', u'-', u'gaisha']

>>> nist.international_tokenize(albb)[:10] == expected_albb
True
>>> nist.international_tokenize(amz)[:10] == expected_amz
True
>>> nist.international_tokenize(rkt)[:10] == expected_rkt
True

# Doctest for patching issue #1926
>>> sent = u'this is a fooâ˜„sentence.'
>>> expected_sent = [u'this', u'is', u'a', u'foo', u'â˜„', u'sentence', u'.']
>>> nist.international_tokenize(sent) == expected_sent
True
z	<skipped>Ú u   â€¨Ú z([\{-\~\[-\` -\&\(-\+\:-\@\/])z \1 z([^0-9])([\.,])z\1 \2 z([\.,])([^0-9])z \1 \2z
([0-9])(-)ÚNumberÚPunctuationÚSymbolz[]^\\-]z\\\g<0>z([ -]+)z([z])([z])c                ó˜   € V P                   w  r#VP                  W14      p\        V4      pV P                  w  r#VP                  W14      pV# )z8Performs the language independent string substituitions.)Ú
STRIP_SKIPÚsubr   ÚSTRIP_EOL_HYPHEN)ÚselfÚtextÚregexpÚsubstitutions   &&  Ú4/usr/lib/python3/dist-packages/nltk/tokenize/nist.pyÚlang_independent_subÚ"NISTTokenizer.lang_independent_sub   sI   € ð
  $Ÿ™ÑˆØ�z‰z˜,Ó-ˆÜ˜DÓ!ˆØ#×4Ñ4ÑˆØ�z‰z˜,Ó-ˆØˆó    c                óŠ  € \        V4      pV P                  V4      pV'       dO   R V,           R ,           pV'       d   VP                  4       pV P                   F  w  rVVP	                  Wa4      pK  	  R P                  VP                  4       4      p\        VP                  4       4      pV'       d   V# VP                  4       # ©r   )Ústrr   ÚlowerÚLANG_DEPENDENT_REGEXESr   ÚjoinÚsplitÚstrip)r   r   Ú	lowercaseÚwestern_langÚ
return_strr   r   s   &&&&&  r   ÚtokenizeÚNISTTokenizer.tokenize‹   s—   € Ü�4‹yˆà×(Ñ(¨Ó.ˆçà˜•: Õ#ˆDßØ—z‘z“|�Ø(,×(CÔ(CÑ$�Ø—z‘z ,Ó5’ñ )Dð �x‰x˜Ÿ
™
›Ó%ˆô �4—:‘:“<Ó ˆß!ˆtÐ3 t§z¡z£|Ð3r   c                ó´  € \        V4      pV P                  w  rVVP                  Wa4      pV P                  w  rVVP                  Wa4      p\	        V4      pV'       d   VP                  4       pV P                   F  w  rVVP                  Wa4      pK  	  R P                  VP                  4       P                  4       4      pV'       d   V# VP                  4       # r   )
r   r   r   r   r   r   ÚINTERNATIONAL_REGEXESr   r   r   )r   r   r    Úsplit_non_asciir"   r   r   s   &&&&&  r   Úinternational_tokenizeÚ$NISTTokenizer.international_tokenizež   s°   € ô �4‹yˆð  $Ÿ™ÑˆØ�z‰z˜,Ó-ˆØ#×4Ñ4ÑˆØ�z‰z˜,Ó-ˆÜ˜DÓ!ˆçØ—:‘:“<ˆDà$(×$>Ô$>Ñ ˆFØ—:‘:˜lÓ1ŠDñ %?ð
 �x‰x˜Ÿ
™
›×*Ñ*Ó,Ó-ˆß!ˆtÐ3 t§z¡z£|Ð3r   © N)FTF)$Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚreÚcompiler   r   ÚPUNCTÚPERIOD_COMMA_PRECEEDÚPERIOD_COMMA_FOLLOWÚDASH_PRECEED_DIGITr   r   r   Úsetr   ÚcharsÚ
pup_numberÚ	pup_punctÚ
pup_symbolr   Únumber_regexÚpunct_regexÚsymbol_regexÚNONASCIIÚPUNCT_1ÚPUNCT_2ÚSYMBOLSr&   r   r#   r(   Ú__static_attributes__Ú__classdictcell__)Ú__classdict__s   @r   r   r      sä  ø‡ € ñ+ð\ —’˜KÓ(¨"Ð,€Jà—z’z (Ó+¨SÐ0Ðà�JŠJÐ8Ó9¸7ÐB€EàŸ:š:Ð&8Ó9¸:ÐEÐàŸ*š*Ð%7Ó8¸*ÐDÐàŸš LÓ1°:Ð=Ðð 	ØØØð	Ðñ �R—W‘W™S ×!3Ò!3°HÓ!=Ó>Ó?Ó@€JÙ�B—G‘G™C × 2Ò 2°=Ó AÓBÓCÓD€IÙ�R—W‘W™S ×!3Ò!3°HÓ!=Ó>Ó?Ó@€Jð —6’6˜* j°*Ó=€LØ—&’&˜ Z°Ó;€KØ—6’6˜* j°*Ó=€Lð �zŠzÐ*Ó+¨WÐ4€Hð 	�
Š
�R˜�~ T¨+¨°bÐ9Ó:Øð€Gð
 	�
Š
�R˜�} D¨¨°bÐ9Ó:Øð€Gð
 �jŠj˜2˜l˜^¨2Ð.Ó/°Ð8€Gà% w°¸ÐAÐò
ô4÷&4ò 4r   r   )
r/   Úior0   Únltk.corpusr   Únltk.tokenize.apir   Únltk.tokenize.utilr   r   r*   r   r   Ú<module>rI      s)   ðñó 
Û 	å $Ý (Ý +ôY4�Jö Y4r   