§
    'ê[f1  ã                   óz   — d Z ddlZddlmZ ddlmZmZmZmZ ddl	m
Z
  G d„ de¦  «        Z G d„ d	e
¦  «        ZdS )
a¸  
Lexical translation model that considers word order.

IBM Model 2 improves on Model 1 by accounting for word order.
An alignment probability is introduced, a(i | j,l,m), which predicts
a source word position, given its aligned target word's position.

The EM algorithm used in Model 2 is:

:E step: In the training data, collect counts, weighted by prior
         probabilities.

         - (a) count how many times a source language word is translated
               into a target language word
         - (b) count how many times a particular position in the source
               sentence is aligned to a particular position in the target
               sentence

:M step: Estimate new probabilities based on the counts from the E step

Notations
---------

:i: Position in the source sentence
     Valid values are 0 (for NULL), 1, 2, ..., length of source sentence
:j: Position in the target sentence
     Valid values are 1, 2, ..., length of target sentence
:l: Number of words in the source sentence, excluding NULL
:m: Number of words in the target sentence
:s: A word in the source language
:t: A word in the target language

References
----------

Philipp Koehn. 2010. Statistical Machine Translation.
Cambridge University Press, New York.

Peter E Brown, Stephen A. Della Pietra, Vincent J. Della Pietra, and
Robert L. Mercer. 1993. The Mathematics of Statistical Machine
Translation: Parameter Estimation. Computational Linguistics, 19 (2),
263-311.
é    N©Údefaultdict)ÚAlignedSentÚ	AlignmentÚIBMModelÚ	IBMModel1)ÚCountsc                   óT   ‡ — e Zd ZdZdˆ fd„	Zd„ Zd„ Zd„ Zd„ Zd„ Z	d	„ Z
d
„ Zd„ Zˆ xZS )Ú	IBMModel2u`  
    Lexical translation model that considers word order

    >>> bitext = []
    >>> bitext.append(AlignedSent(['klein', 'ist', 'das', 'haus'], ['the', 'house', 'is', 'small']))
    >>> bitext.append(AlignedSent(['das', 'haus', 'ist', 'ja', 'groÃŸ'], ['the', 'house', 'is', 'big']))
    >>> bitext.append(AlignedSent(['das', 'buch', 'ist', 'ja', 'klein'], ['the', 'book', 'is', 'small']))
    >>> bitext.append(AlignedSent(['das', 'haus'], ['the', 'house']))
    >>> bitext.append(AlignedSent(['das', 'buch'], ['the', 'book']))
    >>> bitext.append(AlignedSent(['ein', 'buch'], ['a', 'book']))

    >>> ibm2 = IBMModel2(bitext, 5)

    >>> print(round(ibm2.translation_table['buch']['book'], 3))
    1.0
    >>> print(round(ibm2.translation_table['das']['book'], 3))
    0.0
    >>> print(round(ibm2.translation_table['buch'][None], 3))
    0.0
    >>> print(round(ibm2.translation_table['ja'][None], 3))
    0.0

    >>> print(round(ibm2.alignment_table[1][1][2][2], 3))
    0.939
    >>> print(round(ibm2.alignment_table[1][2][2][2], 3))
    0.0
    >>> print(round(ibm2.alignment_table[2][2][4][5], 3))
    1.0

    >>> test_sentence = bitext[2]
    >>> test_sentence.words
    ['das', 'buch', 'ist', 'ja', 'klein']
    >>> test_sentence.mots
    ['the', 'book', 'is', 'small']
    >>> test_sentence.alignment
    Alignment([(0, 0), (1, 1), (2, 2), (3, 2), (4, 3)])

    Nc                 óf  •— t          ¦   «                              |¦  «         |€5t          |d|z  ¦  «        }|j        | _        |                      |¦  «         n|d         | _        |d         | _        t          d|¦  «        D ]}|                      |¦  «         Œ|                      |¦  «         dS )a™  
        Train on ``sentence_aligned_corpus`` and create a lexical
        translation model and an alignment model.

        Translation direction is from ``AlignedSent.mots`` to
        ``AlignedSent.words``.

        :param sentence_aligned_corpus: Sentence-aligned parallel corpus
        :type sentence_aligned_corpus: list(AlignedSent)

        :param iterations: Number of iterations to run training algorithm
        :type iterations: int

        :param probability_tables: Optional. Use this to pass in custom
            probability values. If not specified, probabilities will be
            set to a uniform distribution, or some other sensible value.
            If specified, all the following entries must be present:
            ``translation_table``, ``alignment_table``.
            See ``IBMModel`` for the type and purpose of these tables.
        :type probability_tables: dict[str]: object
        Né   Útranslation_tableÚalignment_tabler   )	ÚsuperÚ__init__r   r   Úset_uniform_probabilitiesr   ÚrangeÚtrainÚ	align_all)ÚselfÚsentence_aligned_corpusÚ
iterationsÚprobability_tablesÚibm1ÚnÚ	__class__s         €úG/var/www/piapp/venv/lib/python3.11/site-packages/nltk/translate/ibm2.pyr   zIBMModel2.__init__c   sÃ   ø€ õ, 	‰Œ×ÒÐ0Ñ1Ô1Ð1àÐ%õ Ð4°a¸*±nÑEÔEˆDØ%)Ô%;ˆDÔ"Ø×*Ò*Ð+BÑCÔCÐCÐCð &8Ð8KÔ%LˆDÔ"Ø#5Ð6GÔ#HˆDÔ å�q˜*Ñ%Ô%ð 	0ð 	0ˆAØ�JŠJÐ.Ñ/Ô/Ð/Ð/à�ŠÐ.Ñ/Ô/Ð/Ð/Ð/ó    c                 óÄ  — t          ¦   «         }|D ]Î}t          |j        ¦  «        }t          |j        ¦  «        }||f|vrž|                     ||f¦  «         d|dz   z  }|t
          j        k     r't          j        dt          |¦  «        z   dz   ¦  «         t          d|dz   ¦  «        D ]4}t          d|dz   ¦  «        D ]}|| j        |         |         |         |<   ŒŒ5ŒÏd S )Né   zA source sentence is too long (z& words). Results may be less accurate.r   )ÚsetÚlenÚmotsÚwordsÚaddr   ÚMIN_PROBÚwarningsÚwarnÚstrr   r   )	r   r   Úl_m_combinationsÚaligned_sentenceÚlÚmÚinitial_probÚiÚjs	            r   r   z#IBMModel2.set_uniform_probabilitiesŒ   s  € å™5œ5ÐØ 7ð 	Hð 	HÐÝÐ$Ô)Ñ*Ô*ˆAÝÐ$Ô*Ñ+Ô+ˆAØ�1ˆvÐ-Ð-Ð-Ø ×$Ò$ a¨ VÑ,Ô,Ð,Ø  A¨¡E™{�Ø¥(Ô"3Ò3Ð3Ý”MØ9Ý˜a™&œ&ñ!àBñCñô ð õ ˜q ! a¡%™œð Hð H�AÝ" 1 a¨!¡e™_œ_ð Hð H˜Ø;G˜Ô,¨QÔ/°Ô2°1Ô5°aÑ8Ð8ðHøð	Hð 	Hr   c           
      óF  — t          ¦   «         }|D ]å}d g|j        z   }dg|j        z   }t          |j        ¦  «        }t          |j        ¦  «        }|                      ||¦  «        }t          d|dz   ¦  «        D ]{}	||	         }
t          d|dz   ¦  «        D ]]}||         }|                      ||	||¦  «        }|||
         z  }|                     |||
¦  «         |                     |||	||¦  «         Œ^Œ|Œæ|  	                    |¦  «         |  
                    |¦  «         d S )NÚUNUSEDr    r   )ÚModel2Countsr#   r$   r"   Úprob_all_alignmentsr   Úprob_alignment_pointÚupdate_lexical_translationÚupdate_alignmentÚ*maximize_lexical_translation_probabilitiesÚ maximize_alignment_probabilities)r   Úparallel_corpusÚcountsr+   Úsrc_sentenceÚtrg_sentencer,   r-   Útotal_countr0   Útr/   ÚsÚcountÚnormalized_counts                  r   r   zIBMModel2.train    sa  € Ý‘”ˆØ /ð 	Jð 	JÐØ ˜6Ð$4Ô$9Ñ9ˆLØ$˜:Ð(8Ô(>Ñ>ˆLÝÐ$Ô)Ñ*Ô*ˆAÝÐ$Ô*Ñ+Ô+ˆAð ×2Ò2°<ÀÑNÔNˆKõ ˜1˜a !™e‘_”_ð Jð J�Ø  ”O�Ý˜q ! a¡%™œð Jð J�AØ$ Qœ�AØ ×5Ò5°a¸¸LÈ,ÑWÔW�EØ',¨{¸1¬~Ñ'=Ð$à×5Ò5Ð6FÈÈ1ÑMÔMÐMØ×+Ò+Ð,<¸aÀÀAÀqÑIÔIÐIÐIðJðJð 	×7Ò7¸Ñ?Ô?Ð?Ø×-Ò-¨fÑ5Ô5Ð5Ð5Ð5r   c                 ó”  — t           j        }|j                             ¦   «         D ]¡\  }}|                     ¦   «         D ]‡\  }}|                     ¦   «         D ]m\  }}|D ]e}	|j        |         |         |         |	         |j        |         |         |	         z  }
t          |
|¦  «        | j        |         |         |         |	<   ŒfŒnŒˆŒ¢d S ©N)r   r&   Ú	alignmentÚitemsÚalignment_for_any_iÚmaxr   )r   r;   r&   r/   Új_sr0   Úsrc_sentence_lengthsr,   Útrg_sentence_lengthsr-   Úestimates              r   r9   z*IBMModel2.maximize_alignment_probabilitiesº   s  € ÝÔ$ˆØÔ&×,Ò,Ñ.Ô.ð 	Sð 	S‰FˆAˆsØ+.¯9ª9©;¬;ð Sð SÑ'�Ð'Ø/C×/IÒ/IÑ/KÔ/Kð Sð SÑ+�AÐ+Ø1ð Sð S˜à"Ô,¨QÔ/°Ô2°1Ô5°aÔ8Ø$Ô8¸Ô;¸AÔ>¸qÔAñBð !õ <?¸xÈÑ;RÔ;R˜Ô,¨QÔ/°Ô2°1Ô5°aÑ8Ð8ðSðSðSð	Sð 	Sr   c                 ó  — t          d„ ¦  «        }t          dt          |¦  «        ¦  «        D ]P}||         }t          dt          |¦  «        ¦  «        D ](}||xx         |                      ||||¦  «        z  cc<   Œ)ŒQ|S )aï  
        Computes the probability of all possible word alignments,
        expressed as a marginal distribution over target words t

        Each entry in the return value represents the contribution to
        the total alignment probability by the target word t.

        To obtain probability(alignment | src_sentence, trg_sentence),
        simply sum the entries in the return value.

        :return: Probability of t for all s in ``src_sentence``
        :rtype: dict(str): float
        c                  ó   — dS ©Ng        © rP   r   r   ú<lambda>z/IBMModel2.prob_all_alignments.<locals>.<lambda>Ô   s   € °3€ r   r    r   )r   r   r"   r5   )r   r<   r=   Úalignment_prob_for_tr0   r?   r/   s          r   r4   zIBMModel2.prob_all_alignmentsÆ   s¦   € õ  +¨;¨;Ñ7Ô7ÐÝ�q�#˜lÑ+Ô+Ñ,Ô,ð 	ð 	ˆAØ˜Q”ˆAÝ˜1�c ,Ñ/Ô/Ñ0Ô0ð ð �Ø$ QÐ'Ð'Ô'¨4×+DÒ+DØ�q˜,¨ñ,ô ,ñ Ð'Ð'Ñ'Ð'ðð $Ð#r   c                 óÐ   — t          |¦  «        dz
  }t          |¦  «        dz
  }||         }||         }| j        |         |         | j        |         |         |         |         z  S )zz
        Probability that position j in ``trg_sentence`` is aligned to
        position i in the ``src_sentence``
        r    )r"   r   r   )	r   r/   r0   r<   r=   r,   r-   r@   r?   s	            r   r5   zIBMModel2.prob_alignment_pointÝ   si   € õ
 �ÑÔ Ñ!ˆÝ�ÑÔ Ñ!ˆØ˜ŒOˆØ˜ŒOˆØÔ% aÔ(¨Ô+¨dÔ.BÀ1Ô.EÀaÔ.HÈÔ.KÈAÔ.NÑNÐNr   c                 óx  — d}t          |j        ¦  «        dz
  }t          |j        ¦  «        dz
  }t          |j        ¦  «        D ]\\  }}|dk    rŒ|j        |         }|j        |         }|| j        |         |         | j        |         |         |         |         z  z  }Œ]t          |t          j	        ¦  «        S )zc
        Probability of target sentence and an alignment given the
        source sentence
        g      ð?r    r   )
r"   r<   r=   Ú	enumeraterE   r   r   rH   r   r&   )	r   Úalignment_infoÚprobr,   r-   r0   r/   Útrg_wordÚsrc_words	            r   Úprob_t_a_given_szIBMModel2.prob_t_a_given_sè   sÇ   € ð
 ˆÝ�Ô+Ñ,Ô,¨qÑ0ˆÝ�Ô+Ñ,Ô,¨qÑ0ˆå˜nÔ6Ñ7Ô7ð 	ð 	‰DˆAˆqØ�AŠvˆvØØ%Ô2°1Ô5ˆHØ%Ô2°1Ô5ˆHØØÔ& xÔ0°Ô:ØÔ& qÔ)¨!Ô,¨QÔ/°Ô2ñ3ñˆDˆDõ
 �4�Ô*Ñ+Ô+Ð+r   c                 ó:   — |D ]}|                       |¦  «         Œd S rD   )Úalign)r   r:   Úsentence_pairs      r   r   zIBMModel2.align_allý   s0   € Ø,ð 	&ð 	&ˆMØ�JŠJ�}Ñ%Ô%Ð%Ð%ð	&ð 	&r   c                 óB  — g }t          |j        ¦  «        }t          |j        ¦  «        }t          |j        ¦  «        D ]Ë\  }}| j        |         d         | j        d         |dz            |         |         z  }t          |t          j        ¦  «        }d}t          |j        ¦  «        D ]H\  }	}
| j        |         |
         | j        |	dz            |dz            |         |         z  }||k    r|}|	}ŒI| 	                    ||f¦  «         ŒÌt          |¦  «        |_        dS )a  
        Determines the best word alignment for one sentence pair from
        the corpus that the model was trained on.

        The best alignment will be set in ``sentence_pair`` when the
        method returns. In contrast with the internal implementation of
        IBM models, the word indices in the ``Alignment`` are zero-
        indexed, not one-indexed.

        :param sentence_pair: A sentence in the source language and its
            counterpart sentence in the target language
        :type sentence_pair: AlignedSent
        Nr   r    )r"   r#   r$   rU   r   r   rH   r   r&   Úappendr   rE   )r   r]   Úbest_alignmentr,   r-   r0   rX   Ú	best_probÚbest_alignment_pointr/   rY   Ú
align_probs               r   r\   zIBMModel2.align  sA  € ð ˆå�Ô"Ñ#Ô#ˆÝ�Ô#Ñ$Ô$ˆå$ ]Ô%8Ñ9Ô9ð 	=ð 	=‰KˆAˆxð Ô& xÔ0°Ô6ØÔ& qÔ)¨!¨a©%Ô0°Ô3°AÔ6ñ7ð õ ˜I¥xÔ'8Ñ9Ô9ˆIØ#'Ð Ý(¨Ô);Ñ<Ô<ð -ð -‘��8àÔ*¨8Ô4°XÔ>ØÔ*¨1¨q©5Ô1°!°a±%Ô8¸Ô;¸AÔ>ñ?ð ð  Ò*Ð*Ø *�IØ+,Ð(øà×!Ò! 1Ð&:Ð";Ñ<Ô<Ð<Ð<å"+¨NÑ";Ô";ˆÔÐÐr   rD   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r9   r4   r5   rZ   r   r\   Ú__classcell__©r   s   @r   r   r   ;   sÊ   ø€ € € € € ð%ð %ðN'0ð '0ð '0ð '0ð '0ð '0ðRHð Hð Hð(6ð 6ð 6ð4
Sð 
Sð 
Sð$ð $ð $ð.	Oð 	Oð 	Oð,ð ,ð ,ð*&ð &ð &ð&<ð &<ð &<ð &<ð &<ð &<ð &<r   r   c                   ó.   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ xZS )r3   zo
    Data object to store counts of various parameters during training.
    Includes counts for alignment.
    c                 óœ   •— t          ¦   «                              ¦   «          t          d„ ¦  «        | _        t          d„ ¦  «        | _        d S )Nc                  ó"   — t          d„ ¦  «        S )Nc                  ó"   — t          d„ ¦  «        S )Nc                  ó"   — t          d„ ¦  «        S )Nc                  ó   — dS rO   rP   rP   r   r   rQ   z]Model2Counts.__init__.<locals>.<lambda>.<locals>.<lambda>.<locals>.<lambda>.<locals>.<lambda>3  s   € ÐPS€ r   r   rP   r   r   rQ   zKModel2Counts.__init__.<locals>.<lambda>.<locals>.<lambda>.<locals>.<lambda>3  s   € ½KÈÈÑ<TÔ<T€ r   r   rP   r   r   rQ   z9Model2Counts.__init__.<locals>.<lambda>.<locals>.<lambda>3  s   € ­Ð4TÐ4TÑ(UÔ(U€ r   r   rP   r   r   rQ   z'Model2Counts.__init__.<locals>.<lambda>3  s   € •KÐ UÐ UÑVÔV€ r   c                  ó"   — t          d„ ¦  «        S )Nc                  ó"   — t          d„ ¦  «        S )Nc                  ó   — dS rO   rP   rP   r   r   rQ   zKModel2Counts.__init__.<locals>.<lambda>.<locals>.<lambda>.<locals>.<lambda>6  s   € ¸C€ r   r   rP   r   r   rQ   z9Model2Counts.__init__.<locals>.<lambda>.<locals>.<lambda>6  s   € ­°K°KÑ(@Ô(@€ r   r   rP   r   r   rQ   z'Model2Counts.__init__.<locals>.<lambda>6  s   € •KÐ @Ð @ÑAÔA€ r   )r   r   r   rE   rG   )r   r   s    €r   r   zModel2Counts.__init__0  sP   ø€ Ý‰Œ×ÒÑÔÐÝ$ØVÐVñ
ô 
ˆŒõ $/ØAÐAñ$
ô $
ˆÔ Ð Ð r   c                 óf   — | j         |         |xx         |z  cc<   | j        |xx         |z  cc<   d S rD   )Ú	t_given_sÚany_t_given_s)r   rA   r@   r?   s       r   r6   z'Model2Counts.update_lexical_translation9  sJ   € ØŒ�qÔ˜!ÐÐÔ Ñ%ÐÐÑØÔ˜1ÐÐÔ Ñ&ÐÐÑÐÐr   c                 ó–   — | j         |         |         |         |xx         |z  cc<   | j        |         |         |xx         |z  cc<   d S rD   )rE   rG   )r   rA   r/   r0   r,   r-   s         r   r7   zModel2Counts.update_alignment=  s^   € ØŒ�qÔ˜!Ô˜QÔ Ð"Ð"Ô" eÑ+Ð"Ð"Ñ"ØÔ  Ô# AÔ& qÐ)Ð)Ô)¨UÑ2Ð)Ð)Ñ)Ð)Ð)r   )rd   re   rf   rg   r   r6   r7   rh   ri   s   @r   r3   r3   *  s`   ø€ € € € € ðð ð

ð 
ð 
ð 
ð 
ð'ð 'ð 'ð3ð 3ð 3ð 3ð 3ð 3ð 3r   r3   )rg   r'   Úcollectionsr   Únltk.translater   r   r   r   Únltk.translate.ibm_modelr	   r   r3   rP   r   r   ú<module>rz      sÊ   ðð*ð *ðX €€€Ø #Ð #Ð #Ð #Ð #Ð #à FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ +Ð +Ð +Ð +Ð +Ð +ðl<ð l<ð l<ð l<ð l<�ñ l<ô l<ð l<ð^3ð 3ð 3ð 3ð 3�6ñ 3ô 3ð 3ð 3ð 3r   