+
    ÖÇÜh]%  ã                   óx   € ^ RI t ^ RIt^ RIt^ RIt^ RIHt ^ RIHtHtH	t	H
t
HtHt ^ RIHt Rt ! R R]4      tR# )é    N)ÚPIPE)Ú_java_optionsÚconfig_javaÚfind_dirÚ	find_fileÚfind_jarÚjava)Ú
TokenizerIz!https://nlp.stanford.edu/softwarec                   óf   a a€ ] tR t^ t oRtRtRR ltR tV 3R ltR t	R t
R tRR	 ltR
tVtV ;t# )ÚStanfordSegmenteru  Interface to the Stanford Segmenter

If stanford-segmenter version is older than 2016-10-31, then path_to_slf4j
should be provieded, for example::

    seg = StanfordSegmenter(path_to_slf4j='/YOUR_PATH/slf4j-api.jar')

>>> from nltk.tokenize.stanford_segmenter import StanfordSegmenter
>>> seg = StanfordSegmenter() # doctest: +SKIP
>>> seg.default_config('zh') # doctest: +SKIP
>>> sent = u'è¿™æ˜¯æ–¯å�¦ç¦�ä¸­æ–‡åˆ†è¯�å™¨æµ‹è¯•'
>>> print(seg.segment(sent)) # doctest: +SKIP
è¿™ æ˜¯ æ–¯å�¦ç¦� ä¸­æ–‡ åˆ†è¯�å™¨ æµ‹è¯•
<BLANKLINE>
>>> seg.default_config('ar') # doctest: +SKIP
>>> sent = u'Ù‡Ø°Ø§ Ù‡Ùˆ ØªØµÙ†ÙŠÙ� Ø³ØªØ§Ù†Ù�ÙˆØ±Ø¯ Ø§Ù„Ø¹Ø±Ø¨ÙŠ Ù„Ù„ÙƒÙ„Ù…Ø§Øª'
>>> print(seg.segment(sent.split())) # doctest: +SKIP
Ù‡Ø°Ø§ Ù‡Ùˆ ØªØµÙ†ÙŠÙ� Ø³ØªØ§Ù†Ù�ÙˆØ±Ø¯ Ø§Ù„Ø¹Ø±Ø¨ÙŠ Ù„ Ø§Ù„ÙƒÙ„Ù…Ø§Øª
<BLANKLINE>
zstanford-segmenter.jarc           	     óR  € \         P                  ! R \        4       \         P                  ! \	        R4      \        ^R7       \         P                  ! R\        4       \        V P                  VR
R\        VR7      pVe   \        RVRR\        VR7      pMRp\        P                  P                  R WÞ3 4       4      V n        W0n        W@n        W`n        Wpn        W€n        WPn        W�n        WÀn        V
f   / MT
p
RP                  R	 V
P)                  4        4       4      V n        R# )Úalwaysz}
The StanfordTokenizer will be deprecated in version 3.2.5.
Please use [91mnltk.parse.corenlp.CoreNLPTokenizer[0m instead.')Ú
stacklevelÚignore)Úenv_varsÚ
searchpathÚurlÚverboseNzslf4j-api.jarc              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5i©N© )Ú.0Ú_s   & ÚB/usr/lib/python3/dist-packages/nltk/tokenize/stanford_segmenter.pyÚ	<genexpr>Ú-StanfordSegmenter.__init__.<locals>.<genexpr>j   s   é € ð -
Ù2�!�AŠAÓ2ùs   ‚Œ
Ú,c              3   ó\   "  € T F"  w  rV R \         P                  ! V4       2x € K$  	  R# 5i)Ú=N)ÚjsonÚdumps)r   ÚkeyÚvals   &  r   r   r   x   s)   é € ð %
Ù7F©8¨3ˆsˆe�1”T—Z’Z “_Ð%Õ&³ùs   ‚*,©ÚSTANFORD_SEGMENTERr   )ÚSLF4Jr%   )ÚwarningsÚsimplefilterÚDeprecationWarningÚwarnÚstrr   Ú_JARÚ_stanford_urlÚosÚpathsepÚjoinÚ_stanford_jarÚ_java_classÚ_modelÚ_sihan_corpora_dictÚ_sihan_post_processingÚ_keep_whitespacesÚ_dictÚ	_encodingÚjava_optionsÚitemsÚ_options_cmd)ÚselfÚpath_to_jarÚpath_to_slf4jÚ
java_classÚpath_to_modelÚpath_to_dictÚpath_to_sihan_corpora_dictÚsihan_post_processingÚkeep_whitespacesÚencodingÚoptionsr   r9   Ústanford_segmenterÚslf4js   &&&&&&&&&&&&&  r   Ú__init__ÚStanfordSegmenter.__init__8   s  € ô  	×Ò˜hÔ(:Ô;Ü�ŠÜðZóô
 Øõ	
ô 	×Ò˜hÔ(:Ô;ä%Ø�I‰IØØ,ØÜØô
Ðð Ò$ÜØØØ8ØÜ!Øô‰Eð ˆEô  ŸZ™ZŸ_™_ñ -
Ø*Ñ2ó-
ó 
ˆÔð &ÔØ#ŒØ#=Ô Ø&;Ô#Ø!1ÔØ!Œ
à!ŒØ(ÔØš‘"¨WˆØŸH™Hñ %
Ø7>·}±}´ó%
ó 
ˆÖó    c                ó,  € Rp\         P                  P                  R4      '       d?   \         P                  P	                  \         P                  P                  R4      R4      0pRV n        RV n        RV n        VR8X  d   RV n        RpM|VR8X  dh   R	V n        R
pRV n        Rp \        VV\        RRR7      V n        Rp \        T\        RRR7      p\         P                  P	                  Yv4      T n        M\        RV 24      h \        VV\        RRR7      V n        R#   \         d   p\        RT,          4      ThRp?ii ; i  \         d   p\        RT,          4      ThRp?ii ; i  \         d   p\        RT,          4      ThRp?ii ; i)z‘
Attempt to initialize Stanford Word Segmenter for the specified language
using the STANFORD_SEGMENTER and STANFORD_MODELS environment variables
r%   ÚdataNÚfalseÚarz=edu.stanford.nlp.international.arabic.process.ArabicSegmenterz'arabic-segmenter-atb+bn+arztrain.ser.gzÚzhz%edu.stanford.nlp.ie.crf.CRFClassifierzpku.gzÚtruezdict-chris6.ser.gzF)r   r   r   r   z_Could not find '%s' (tried using env. variables STANFORD_MODELS and <STANFORD_SEGMENTER>/data/)z./data/)r   r   r   zMCould not find '%s' (tried using the STANFORD_SEGMENTER environment variable)zUnsupported language r   )ÚSTANFORD_MODELSr$   )rR   r%   )r.   ÚenvironÚgetÚpathr0   r7   r4   r5   r2   r   r-   ÚLookupErrorr   r3   )r<   ÚlangÚsearch_pathÚmodelrA   ÚeÚ	sihan_dirÚpath_to_sihan_dirs   &&      r   Údefault_configÚ StanfordSegmenter.default_config|   sº  € ð ˆÜ�:‰:�>‰>Ð.×/Ò/ÜŸ7™7Ÿ<™<¬¯
©
¯©Ð7KÓ(LÈfÓUÐVˆKð ˆŒ
Ø#'ˆÔ Ø&-ˆÔ#à�4Œ<àOð Ôð >‰Eà�TŒ\ØFˆDÔØˆEØ*0ˆDÔ'à/ˆLðÜ&Ø Ø*Ü%Ø!Ø1ô�”
ð "ˆIðÜ$,ØÜ%Ø!Ø4ô	%Ð!ô ,.¯7©7¯<©<Ð8IÓ+U�Õ(ô Ð 5°d°VÐ<Ó=Ð=ð	Ü#ØØ&Ü!ØØBôˆDŽKøô3 ô Ü!ðPà"õ#óð ð	ûðûô  ô Ü!ð?ØAJõKóð ðûðûô  ô 	ÜðLØNSõTóð ðûð	úsH   Â&D$ Ã7E
 Ä	E0 Ä$EÄ/EÅEÅ
E-ÅE(Å(E-Å0FÅ;FÆFc                ó&   <€ \         SV `  V4       R # r   )ÚsuperÚtokenize)r<   ÚsÚ	__class__s   &&€r   ra   ÚStanfordSegmenter.tokenizeÅ   s   ø€ Ü‰Ñ˜ÖrK   c                óü   € V P                   RV P                  RV P                  RV.pV P                  e6   VP	                  RV P
                  RV P                  RV P                  .4       V P                  V4      pV# )Ú ú-loadClassifierú-keepAllWhitespacesú	-textFileú-serDictionaryú-sighanCorporaDictú-sighanPostProcessing)r2   r3   r6   r4   Úextendr7   r5   Ú_execute)r<   Úinput_file_pathÚcmdÚstdouts   &&  r   Úsegment_fileÚStanfordSegmenter.segment_fileÈ   s†   € ð ×ÑØØ�K‰KØ!Ø×"Ñ"ØØð
ˆð ×#Ñ#Ò/Ø�J‰Jà$Ø—J‘JØ(Ø×,Ñ,Ø+Ø×/Ñ/ðô	ð —‘˜sÓ#ˆàˆrK   c                ó&   € V P                  V.4      # r   )Úsegment_sents)r<   Útokenss   &&r   ÚsegmentÚStanfordSegmenter.segmentã   s   € Ø×!Ñ! 6 (Ó+Ð+rK   c                ó¢  € V P                   p\        P                  ! RR7      w  q0n        \        P
                  ! VR4      pRP                  R V 4       4      p\        V\        4      '       d   V'       d   VP                  V4      pVP                  V4       VP                  4        V P                  RV P                  RV P                  RV P                  .pV P                  e6   VP!                  R	V P"                  R
V P                  RV P$                  .4       V P'                  V4      p\        P(                  ! V P                  4       V# )rf   T)ÚtextÚwbÚ
c              3   óD   "  € T F  pR P                  V4      x € K  	  R# 5i)Ú N)r0   )r   Úxs   & r   r   Ú2StanfordSegmenter.segment_sents.<locals>.<genexpr>î   s   é € Ð:±	¨1˜3Ÿ8™8 AŸ;˜;³	ùs   ‚ rg   rh   ri   rj   rk   rl   )r8   ÚtempfileÚmkstempÚ_input_file_pathr.   Úfdopenr0   Ú
isinstancer+   ÚencodeÚwriteÚcloser2   r3   r6   r4   rm   r7   r5   rn   Úunlink)r<   Ú	sentencesrE   Ú	_input_fhÚ_inputrp   rq   s   &&     r   ru   ÚStanfordSegmenter.segment_sentsæ   s  € à—>‘>ˆä+3×+;Ò+;ÀÔ+FÑ(ˆ	Ô(ô —I’I˜i¨Ó.ˆ	Ø—‘Ñ:±	Ó:Ó:ˆÜ�fœc×"Ò"§xØ—]‘] 8Ó,ˆFØ�‰˜ÔØ�‰Ôð ×ÑØØ�K‰KØ!Ø×"Ñ"ØØ×!Ñ!ð
ˆð ×#Ñ#Ò/Ø�J‰Jà$Ø—J‘JØ(Ø×,Ñ,Ø+Ø×/Ñ/ðô	ð —‘˜sÓ#ˆô 	�	Š	�$×'Ñ'Ô(àˆrK   c                ó~  € V P                   pVP                  R V.4       V P                  pV'       d   VP                  RV P                  .4       RP                  \        4      p\        V P                  VR7       \        WP                  \        \        R7      w  rgVP                  V4      p\        VRR7       V# )z-inputEncodingz-optionsr~   )rF   r   )Ú	classpathrq   ÚstderrF)r8   rm   r;   r0   r   r   r9   r	   r1   r   Údecode)r<   rp   r   rE   r;   Údefault_optionsrq   Ú_stderrs   &&&     r   rn   ÚStanfordSegmenter._execute  sœ   € Ø—>‘>ˆØ�
‰
Ð$ hÐ/Ô0Ø×(Ñ(ˆßØ�J‰J˜
 D×$5Ñ$5Ð6Ô7àŸ(™(¤=Ó1ˆô 	˜D×-Ñ-°wÕ?äØ×-Ñ-´dÄ4ô
‰ˆð —‘˜xÓ(ˆô 	˜O°UÕ;àˆrK   )r7   r8   rƒ   r2   r6   r3   r;   r4   r5   r1   r9   )NNNNNNrN   rN   zUTF-8NFz-mx2g)F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r,   rI   r]   ra   rr   rw   ru   rn   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)rc   Ú__classdict__s   @@r   r   r       s>   ù‡ € ñð* $€DôB
òHGõRòò6,ò(÷Tô rK   r   )r    r.   r�   r'   Ú
subprocessr   Únltk.internalsr   r   r   r   r   r	   Únltk.tokenize.apir
   r-   r   r   rK   r   Ú<module>r¡      s8   ðó Û 	Û Û Ý ÷÷ õ )à3€ôD˜
ö DrK   