+
    ÖÇÜh!  ã                   óÀ   € R t ^ RIt ^ RIHt ^ RIHt ]P                  ! ^4      t
 ! R R4      tR	 tR
 t]3R lt]3R ltR tR tR#   ] d    R tR tR t LHi ; i)zž

A port of the Gale-Church Aligner.

Gale & Church (1993), A Program for Aligning Sentences in Bilingual Corpora.
https://aclweb.org/anthology/J93-1004.pdf

N)Úlogsf)Únormc                ó¾  € \        V 4      p^^RV,          ,           ,          pV\        P                  ! V) V,          R,
          VRVRVRVRVRVRVRVRVR,          ,           ,          ,           ,          ,           ,          ,           ,          ,           ,          ,           ,          ,           ,          ,           ,          ,           4      ,          pV R	8¼  d   V# R
V,
          # )zComplementary error function.ç      à?gÅ›ˆÀ‰?ô?gˆ5�Ô ð?gê`ygñ×?gðÃÞÙÆ¸?gæ#vùØÑ?g¾¡SöÐ÷?g¤¥v¯(ßÅ?g        g       @g¥êª‚IØÇ¿g€9¦Ë)ò¿gýÞ…1Oê¿)ÚabsÚmathÚexp)ÚxÚzÚtÚrs   &   Ú</usr/lib/python3/dist-packages/nltk/translate/gale_church.pyÚerfccr      sß   € ä�‹FˆØ��S˜1•W•ÕˆØ”—’ØˆB��FØõààØàØà"Øà'Øà *Ø"#à$/Ø&'Ø'1°A¸ÀqÈ:Å~Õ9UÕ4VÕ'Võ'Xõ%Xõ#"õ!"õ	õ
õõõõõõõõó
õ 
ˆð< �Œ8ØˆHà˜•7ˆNó    c                ój   € ^R\        V \        P                  ! ^4      ,          4      ,          ,
          # )u>   Return the area under the normal distribution from M{-âˆž..x}.r   )r   r   Úsqrt©r	   s   &r   Únorm_cdfr   @   s$   € à�3œ˜q¤4§9¢9¨Q£<Õ/Ó0Õ0Õ0Ð0r   c                 óˆ   €  \         P                  ! ^\        V 4      ,
          4      #   \         d    \	        R4      u # i ; i)é   ú-inf)r   Úlogr   Ú
ValueErrorÚfloatr   s   &r   Ú
norm_logsfr   D   s7   € ð	!Ü—8’8˜A¤¨£�OÓ,Ð,øÜô 	!Ü˜“=Ò ð	!ús   ‚%( ¨AÁ Ac                   ó:   € ] tR t^NtRRR	RR
RRRRRRR/t^tRtRtR# )ÚLanguageIndependentgõÛ×�sF„?g{®Gázì?gbX9´È¶?gºI+‡†?g333333@© N)r   é    )r   r   )r   r   )é   r   )r   r   )r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚPRIORSÚAVERAGE_CHARACTERSÚVARIANCE_CHARACTERSÚ__static_attributes__r   r   r   r   r   N   s7   † ð 	�Ø�Ø�Ø�Ø�Ø�ð€Fð ÐØÔr   r   c                óN  € . p\        V4      \        V4      3pVR8w  dÉ   \        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       d�    W,          w  rV\        T4       FR  p\        T4       F@  pTP	                  T^ ,          T,
          ^,
          T^,          T,
          ^,
          34       KB  	  KT  	  T^ ,          T,
          T^,          T,
          3pKÏ  VRRR1,          #   \         d&    T^ ,          ^,
          T^,          ^,
          3p EK
  i ; i)a¼  
Traverse the alignment cost from the tracebacks and retrieves
appropriate sentence pairs.

:param backlinks: A dictionary where the key is the alignment points and value is the cost (referencing the LanguageIndependent.PRIORS)
:type backlinks: dict
:param source_sents_lens: A list of target sentences' lengths
:type source_sents_lens: list(int)
:param target_sents_lens: A list of target sentences' lengths
:type target_sents_lens: list(int)
c              3   ó*   "  € T F	  q^ 8¬  x € K  	  R# 5i)r   Nr   )Ú.0Úps   & r   Ú	<genexpr>Útrace.<locals>.<genexpr>n   s   é € Ð$>±X°¨!¦V³Xùs   ‚FTN)r   r   éÿÿÿÿ)ÚlenÚallÚ	TypeErrorÚrangeÚappend)	Ú	backlinksÚsource_sents_lensÚtarget_sents_lensÚlinksÚpositionÚsr   ÚiÚjs	   &&&      r   Útracer<   `   s÷   € ð €EÜÐ%Ó&¬Ð,=Ó(>Ð?€HØ
�fÔ
§£Ñ$>±XÓ$>§§¢Ñ$>±XÓ$>×!>Ò!>ð	ØÕ&‰DˆAô �q–ˆAÜ˜1–X�Ø—‘˜h q�k¨A�o°Õ1°8¸Aµ;Àµ?ÀQÕ3FÐGÖHó ñ ð ˜Q•K !•O X¨a¥[°1¥_Ð5Šà‘�2��;Ðøô ô 	Ø  � a�¨°!­°q­Ð9ˆHÛð	ús   Á
C4 Ã4+D$Ä#D$c                ó<  a aaa€ \        V V3R l\        V^ ,          4       4       4      p\        VV3R l\        V^,          4       4       4      p WgVP                  ,          ,           ^,          pWeP                  ,          V,
          \        P                  ! W…P
                  ,          4      ,          p	\        \        \        T	4      4      ,           \        P                  ! TP                  T,          4      ,           ) #   \         d    \        R4      u # i ; i)a,  Returns the log probability of the two sentences C{source_sents[i]}, C{target_sents[j]}
being aligned with a specific C{alignment}.

@param i: The offset of the source sentence.
@param j: The offset of the target sentence.
@param source_sents: The list of source sentence lengths.
@param target_sents: The list of target sentence lengths.
@param alignment: The alignment type, a tuple of two integers.
@param params: The sentence alignment parameters.

@returns: The log probability of a specific alignment between the two sentences, given the parameters.
c              3   óR   <"  € T F  pSSV,
          ^,
          ,          x € K  	  R# 5i©r   Nr   )r*   Úoffsetr:   Úsource_sentss   & €€r   r,   Ú!align_log_prob.<locals>.<genexpr>‰   ó#   øé € ÐMÑ9L¨vˆl˜1˜v�:¨�>×*Ò*Ó9Lùó   ƒ$'c              3   óR   <"  € T F  pSSV,
          ^,
          ,          x € K  	  R# 5ir?   r   )r*   r@   r;   Útarget_sentss   & €€r   r,   rB   Š   rC   rD   r   )Úsumr2   r%   r   r   r&   ÚZeroDivisionErrorr   ÚLOG2r   r   r   r$   )
r:   r;   rA   rF   Ú	alignmentÚparamsÚl_sÚl_tÚmÚdeltas
   ffff&&    r   Úalign_log_probrP   |   sÐ   û€ ô ÕM¼¸yÈ½|Ô9LÓMÓ
M€CÜ
ÕM¼¸yÈ½|Ô9LÓMÓ
M€Cðð ˜×2Ñ2Õ2Õ2°aÕ7ˆØ×0Ñ0Õ0°3Õ6¼$¿)º)Ø×*Ñ*Õ*ó;
õ 
ˆô ”Jœs 5›zÓ*Õ*¬T¯XªX°f·m±mÀIÕ6NÓ-OÕOÐPÐPøô ô Ü�V‹}Òðús   ÁA$D ÄDÄDc                ó°  € \        VP                  P                  4       4      p. .p/ p\        \	        V 4      ^,           4       EF  p\        \	        V4      ^,           4       F®  p\        R4      pRp	V Fi  p
RV
^ ,          ,
          pWz^,          ,
          pV\	        V4      ) 8  g   V^ 8  d   K<  WK,          V,          \        WgWW¢4      ,           pWØ8  g   Ke  TpT
p	Kk  	  V\        R4      8X  d   ^ pW•Wg3&   VR,          P                  V4       K°  	  \	        V4      ^8”  d   VP                  ^ 4       VP                  . 4       EK  	  \        WPV4      # )am  Return the sentence alignment of two text blocks (usually paragraphs).

    >>> align_blocks([5,5,5], [7,7,7])
    [(0, 0), (1, 1), (2, 2)]
    >>> align_blocks([10,5,5], [12,20])
    [(0, 0), (1, 1), (2, 1)]
    >>> align_blocks([12,20], [10,5,5])
    [(0, 0), (1, 1), (1, 2)]
    >>> align_blocks([10,2,10,10,2,10], [12,3,20,3,12])
    [(0, 0), (1, 1), (2, 2), (3, 2), (4, 3), (5, 4)]

@param source_sents_lens: The list of source sentence lengths.
@param target_sents_lens: The list of target sentence lengths.
@param params: the sentence alignment parameters.
@return: The sentence alignments, a list of index pairs.
ÚinfNr.   )
Úlistr$   Úkeysr2   r/   r   rP   r3   Úpopr<   )r5   r6   rK   Úalignment_typesÚDr4   r:   r;   Úmin_distÚ	min_alignÚaÚprev_iÚprev_jr+   s   &&&           r   Úalign_blocksr]   ˜   s5  € ô$ ˜6Ÿ=™=×-Ñ-Ó/Ó0€Oð 
ˆ€Aà€Iä”3Ð(Ó)¨AÕ-×.ˆÜ”sÐ,Ó-°Õ1Ö2ˆAÜ˜U“|ˆHØˆIÛ$�Ø˜a �d��Ø˜q�T��ØœS ›V˜GÔ# v°¤zÙØ•I˜fÕ%¬ØÐ+Àó)õ �ð –<Ø �HØ !’Iñ %ð œ5 ›<Ô'Ø�à )�q�fÑØˆb�E�L‰L˜Ö"ñ' 3ô* ˆq‹6�AŒ:Ø�E‰E�!ŒHØ	�‰��ñ1 /ô4 �Ð/@ÓAÐAr   c           	     ó¨   € \        V 4      \        V4      8w  d   \        R4      h\        W4       UUu. uF  w  r4\        W4V4      NK  	  upp# u uppi )a×  Creates the sentence alignment of two texts.

Texts can consist of several blocks. Block boundaries cannot be crossed by sentence
alignment links.

Each block consists of a list that contains the lengths (in characters) of the sentences
in this block.

@param source_blocks: The list of blocks in the source text.
@param target_blocks: The list of blocks in the target text.
@param params: the sentence alignment parameters.

@returns: A list of sentence alignment lists
z>Source and target texts do not have the same number of blocks.)r/   r   Úzipr]   )Úsource_blocksÚtarget_blocksrK   Úsource_blockÚtarget_blocks   &&&  r   Úalign_textsrd   Î   s\   € ô ˆ=ÓœS Ó/Ô/ÜØLó
ð 	
ô +.¨mÔ*Kôá*KÑ&ˆLô 	�\°Ö8Ù*Kòð ùó s   ³Ac              #  óR   a a"  € V V3R lp V! S P                  4       4      x € K  5i)zÑSplits an iterator C{it} at values of C{split_value}.

Each instance of C{split_value} is swallowed. The iterator produces
subiterators which need to be consumed fully before the next subiterator
can be used.
c              3   óN   <"  € T pVS8w  d   Vx € SP                  4       pK  R # 5i©N©Únext)ÚfirstÚvÚitÚsplit_values   & €€r   Ú_chunk_iteratorÚ!split_at.<locals>._chunk_iteratoró   s(   øé € ØˆØ�;ÔØŠGØ—‘“	ŠAñ ùs   ƒ"%rh   )rl   rm   rn   s   ff r   Úsplit_atrp   ë   s!   ùé € öð Ù˜bŸg™g›iÓ(Ô(ùs   „#'c                ó¤   € \        W4       UUu. uF.  p\        W14       Uu. uF  p\        R V 4       4      NK  	  upNK0  	  upp# u upi u uppi )z®Parses a stream of tokens and splits it into sentences (using C{soft_delimiter} tokens)
and blocks (using C{hard_delimiter} tokens) for use with the L{align_texts} function.
c              3   ó8   "  € T F  p\        V4      x € K  	  R # 5irg   )r/   )r*   Útokens   & r   r,   Ú%parse_token_stream.<locals>.<genexpr>  s   é € Ð4©˜u”�E—
�
«ùs   ‚)rp   rG   )ÚstreamÚsoft_delimiterÚhard_delimiterÚblock_itÚsentence_its   &&&  r   Úparse_token_streamrz   ý   s_   € ô ! Ô8ôñ
 9ˆHô  (¨ÔAó	
áA�ô Ñ4©Ó4Ö4ÙAô	
ñ 9òð ùò	
ùós   �A¡A¼AÁA)Ú__doc__r   r   r   r   Úscipy.statsÚImportErrorr   r   r   rI   r   r<   rP   r]   rd   rp   rz   r   r   r   Ú<module>r~      s„   ðñó ð4!Ý(Ý ðj ‡x‚x�ƒ{€÷ñ ò$ò8Qð8 ?Rô 3Bðl 6Iô ò:)ô$
øðM ô 1!ò%òN1ô!ð[1!ús   ˆA
 Á
AÁA