+
    ÖÇÜh¬  ã                   ó„   € R t ^ RIHt ^ RIHt  ^ RIt ! R R4      tR t	]
R8X  d
   ]	! 4        R# R#   ] d    Rt L*i ; i)a˜  
A module for language identification using the TextCat algorithm.
An implementation of the text categorization algorithm
presented in Cavnar, W. B. and J. M. Trenkle,
"N-Gram-Based Text Categorization".

The algorithm takes advantage of Zipf's law and uses
n-gram frequencies to profile languages and text-yet to
be identified-then compares using a distance measure.

Language n-grams are provided by the "An Crubadan"
project. A corpus reader was created separately to read
those files.

For details regarding the algorithm, see:
https://www.let.rug.nl/~vannoord/TextCat/textcat.pdf

For details about An Crubadan, see:
https://borel.slu.edu/crubadan/index.html
)Úmaxsize)ÚtrigramsNc                   óX   a € ] tR t^/t o Rt/ tRtRt/ tR t	R t
R tR tR tR	 tR
tV tR# )ÚTextCatNÚ<Ú>c                óÄ   € \         '       g   \        R 4      h^ RIHp Wn        V P                  P                  4        F  pV P                  P                  V4       K   	  R# )z›classify.textcat requires the regex module that supports unicode. Try '$ pip install regex' and see https://pypi.python.org/pypi/regex for further details.)ÚcrubadanN)ÚreÚOSErrorÚnltk.corpusr	   Ú_corpusÚlangsÚ	lang_freq)Úselfr	   Úlangs   &  Ú7/usr/lib/python3/dist-packages/nltk/classify/textcat.pyÚ__init__ÚTextCat.__init__7   sL   € ß‹rÜð#óð õ 	)àŒà—L‘L×&Ñ&Ö(ˆDØ�L‰L×"Ñ" 4Ö(ó )ó    c                ó2   € \         P                  ! RRV4      # )z)Get rid of punctuation except apostrophesz[^\P{P}\']+Ú )r
   Úsub©r   Útexts   &&r   Úremove_punctuationÚTextCat.remove_punctuationG   s   € ä�vŠv�n b¨$Ó/Ð/r   c                ó^  € ^ RI HpHp V P                  V4      pV! V4      pV! 4       pV Fy  p\	        V P
                  V,           V P                  ,           4      pV U	u. uF  p	RP                  V	4      NK  	  p
p	V
 F#  pW¶9   d   Wk;;,          ^,          uu&   K  ^Wk&   K%  	  K{  	  V# u up	i )z'Create FreqDist of trigrams within text)ÚFreqDistÚword_tokenizer   )Únltkr   r   r   r   Ú_START_CHARÚ	_END_CHARÚjoin)r   r   r   r   Ú
clean_textÚtokensÚfingerprintÚtÚtoken_trigram_tuplesÚtriÚtoken_trigramsÚcur_trigrams   &&          r   ÚprofileÚTextCat.profileK   s¡   € ç0à×,Ñ,¨TÓ2ˆ
Ù˜zÓ*ˆá“jˆÛˆAÜ#+¨D×,<Ñ,<¸qÕ,@À4Ç>Á>Õ,QÓ#RÐ Ù6JÓKÑ6J¨s˜bŸg™g cžlÑ6JˆNÐKã-�ØÔ-Ø×,°Õ1Õ,à/0�KÓ,ó	  .ñ	 ð Ðùò Ls   ÁB*c                ó  € V P                   P                  V4      p^ pW$9   dd   \        VP                  4       4      P	                  V4      p\        VP                  4       4      P	                  V4      p\        Wg,
          4      pV# \        pV# )z_Calculate the "out-of-place" measure between the
text and language profile for a single trigram)r   r   ÚlistÚkeysÚindexÚabsr   )r   r   ÚtrigramÚtext_profileÚlang_fdÚdistÚidx_lang_profileÚidx_texts   &&&&    r   Ú	calc_distÚTextCat.calc_dist_   s€   € ð —,‘,×(Ñ(¨Ó.ˆØˆàÔÜ# G§L¡L£NÓ3×9Ñ9¸'ÓBÐÜ˜L×-Ñ-Ó/Ó0×6Ñ6°wÓ?ˆHô Ð'Õ2Ó3ˆDð ˆô ˆDàˆr   c                óÐ   € / pV P                  V4      pV P                  P                  P                  4        F*  p^ pV F  pWPP	                  WFV4      ,          pK  	  WRV&   K,  	  V# )zGCalculate the "out-of-place" measure between
the text and all languages)r,   r   Ú_all_lang_freqr0   r9   )r   r   Ú	distancesr,   r   Ú	lang_distr3   s   &&     r   Ú
lang_distsÚTextCat.lang_distst   si   € ð ˆ	Ø—,‘,˜tÓ$ˆà—L‘L×/Ñ/×4Ñ4Ö6ˆDð ˆIÛ"�ØŸ^™^¨D¸7ÓCÕC’	ñ #ð (�d‹Oñ 7ð Ðr   c                ó„   € V P                  V4      V n        \        V P                  V P                  P                  R7      # )zQFind the language with the min distance
to the text and return its ISO 639-3 code)Úkey)r?   Úlast_distancesÚminÚgetr   s   &&r   Úguess_languageÚTextCat.guess_language†   s4   € ð #Ÿo™o¨dÓ3ˆÔä�4×&Ñ&¨D×,?Ñ,?×,CÑ,CÔDÐDr   )r   rC   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Úfingerprintsr!   r"   rC   r   r   r,   r9   r?   rF   Ú__static_attributes__Ú__classdictcell__)Ú__classdict__s   @r   r   r   /   sC   ø‡ € Ø€GØ€LØ€KØ€Ià€Nò)ò 0òò(ò*÷$Eð Er   r   c                  óV  € ^ RI Hp  . ROpRRRRRRRR	R
RRRRRRRRR/	p\        4       pV Fõ  pV P                  V4      p\	        V4      ^,
          p\        \        \        V4      4      pRp\        ^ V4       FP  p	RRP                  \        ^ Wy,          4       U
u. uF  q¥V	,          V
,          NK  	  up
4      ,           pW‹,          pKR  	  \        RVR,          ,           R,           4       VP                  V4      p\        RV RW,,           R24       \        R4       K÷  	  R# u up
i )é    )ÚudhrÚkmrzNorthern KurdishÚabkÚ	AbkhazianÚpeszIranian PersianÚhinÚHindiÚhawÚHawaiianÚrusÚRussianÚvieÚ
VietnameseÚsrpÚSerbianÚepoÚ	Esperantor   Ú zLanguage snippet: :rQ   éŒ   Nz...zLanguage detection: z (Ú)N)	zKurdish-UTF8zAbkhaz-UTF8zFarsi_Persian-UTF8z
Hindi-UTF8zHawaiian-UTF8zRussian-UTF8zVietnamese-UTF8zSerbian_Srpski-UTF8zEsperanto-UTF8zŒ############################################################################################################################################)r   rR   r   ÚsentsÚlenr/   ÚmapÚranger#   ÚprintrF   )rR   r   ÚfriendlyÚtcÚcur_langÚraw_sentencesÚrowsÚcolsÚsampleÚiÚjÚcur_sentÚguesss                r   Údemorv   �   s.  € Ý ò
€Eð 	Ð!Øˆ{ØÐ ØˆwØˆzØˆyØˆ|ØˆyØˆ{ð
€Hô 
‹€BãˆàŸ
™
 8Ó,ˆÜ�=Ó! AÕ%ˆÜ”Cœ˜]Ó+Ó,ˆàˆô �q˜$–ˆAØ˜SŸX™XÄEÈ!ÈTÍWÔDUÓ&VÑDU¸q°QÕ'7¸×':Ð':ÑDUÑ&VÓWÕWˆHØÕŠFñ  ô
 	Ð" V¨E¥]Õ2°UÕ:Ô;Ø×!Ñ! &Ó)ˆÜÐ$ U G¨2¨h­oÐ->¸aÐ@ÔAÜˆiÖó# ùò 'Ws   ÂD&Ú__main__)Ú__doc__Úsysr   Ú	nltk.utilr   Úregexr
   ÚImportErrorr   rv   rH   © r   r   Ú<module>r~      sZ   ðñõ* å ðÛ÷\Eñ \Eò@.ðb ˆzÔÙ†Fñ øðq ô Ø	‚Bðús   �3 ³	?¾?