§
    'ê[f‰  ã                   óˆ   — d Z ddlmZ ddlmZ ddlmZ d„ Z G d„ de¦  «        Z G d„ d	e¦  «        Z	 G d
„ de¦  «        Z
dS )z…Smoothing algorithms for language modeling.

According to Chen & Goodman 1995 these should work with both Backoff and
Interpolation.
é    )Úmethodcaller)Ú	Smoothing)ÚConditionalFreqDistc                 ó¬   ‡— t          | t          ¦  «        rt          d¦  «        nd„ Št          ˆfd„|                      ¦   «         D ¦   «         ¦  «        S )zµCount values that are greater than zero in a distribution.

    Assumes distribution is either a mapping with counts as values or
    an instance of `nltk.ConditionalFreqDist`.
    ÚNc                 ó   — | S ©N© )Úcounts    úE/var/www/piapp/venv/lib/python3.11/site-packages/nltk/lm/smoothing.pyú<lambda>z'_count_values_gt_zero.<locals>.<lambda>   s   € ˜5€ ó    c              3   ó:   •K  — | ]} ‰|¦  «        d k    ¯dV — ŒdS )r   é   Nr
   )Ú.0Údist_or_countÚas_counts     €r   ú	<genexpr>z(_count_values_gt_zero.<locals>.<genexpr>   sA   øè è € ð ð Ø¸¸ÀÑ8OÔ8OÐRSÒ8SÐ8SˆÐ8SÐ8SÐ8SÐ8Sðð r   )Ú
isinstancer   r   ÚsumÚvalues)Údistributionr   s    @r   Ú_count_values_gt_zeror      sv   ø€ õ �lÕ$7Ñ8Ô8ð	!��SÑÔÐà Ð ð õ ð ð ð ð Ø+×2Ò2Ñ4Ô4ðñ ô ñ ô ð r   c                   ó4   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zd„ Zˆ xZS )Ú
WittenBellzWitten-Bell smoothing.c                 ó>   •—  t          ¦   «         j        ||fi |¤Ž d S r	   )ÚsuperÚ__init__)ÚselfÚ
vocabularyÚcounterÚkwargsÚ	__class__s       €r   r   zWittenBell.__init__'   s*   ø€ Ø�‰ŒÔ˜ WÐ7Ð7°Ð7Ð7Ð7Ð7Ð7r   c                 ó€   — | j         |                              |¦  «        }|                      |¦  «        }d|z
  |z  |fS )Ng      ð?)ÚcountsÚfreqÚ_gamma©r   ÚwordÚcontextÚalphaÚgammas        r   Úalpha_gammazWittenBell.alpha_gamma*   sA   € Ø”˜GÔ$×)Ò)¨$Ñ/Ô/ˆØ—’˜GÑ$Ô$ˆØ�e‘˜uÑ$ eÐ+Ð+r   c                 ó€   — t          | j        |         ¦  «        }||| j        |                              ¦   «         z   z  S r	   )r   r%   r   ©r   r*   Ún_pluss      r   r'   zWittenBell._gamma/   s9   € Ý& t¤{°7Ô';Ñ<Ô<ˆØ˜ $¤+¨gÔ"6×"8Ò"8Ñ":Ô":Ñ:Ñ;Ð;r   c                 ó@   — | j         j                             |¦  «        S r	   ©r%   Úunigramsr&   ©r   r)   s     r   Úunigram_scorezWittenBell.unigram_score3   ó   € ØŒ{Ô#×(Ò(¨Ñ.Ô.Ð.r   ©	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r-   r'   r5   Ú__classcell__©r#   s   @r   r   r   $   sk   ø€ € € € € Ø Ð ð8ð 8ð 8ð 8ð 8ð,ð ,ð ,ð
<ð <ð <ð/ð /ð /ð /ð /ð /ð /r   r   c                   ó6   ‡ — e Zd ZdZdˆ fd„	Zd„ Zd„ Zd„ Zˆ xZS )ÚAbsoluteDiscountingz!Smoothing with absolute discount.ç      è?c                 óL   •—  t          ¦   «         j        ||fi |¤Ž || _        d S r	   )r   r   Údiscount)r   r    r!   rB   r"   r#   s        €r   r   zAbsoluteDiscounting.__init__:   s/   ø€ Ø�‰ŒÔ˜ WÐ7Ð7°Ð7Ð7Ð7Ø ˆŒˆˆr   c                 óÆ   — t          | j        |         |         | j        z
  d¦  «        | j        |                              ¦   «         z  }|                      |¦  «        }||fS )Nr   )Úmaxr%   rB   r   r'   r(   s        r   r-   zAbsoluteDiscounting.alpha_gamma>   s_   € å�”˜GÔ$ TÔ*¨T¬]Ñ:¸AÑ>Ô>ØŒk˜'Ô"×$Ò$Ñ&Ô&ñ'ð 	ð —’˜GÑ$Ô$ˆØ�eˆ|Ðr   c                 óŠ   — t          | j        |         ¦  «        }| j        |z  | j        |                              ¦   «         z  S r	   )r   r%   rB   r   r/   s      r   r'   zAbsoluteDiscounting._gammaF   s;   € Ý& t¤{°7Ô';Ñ<Ô<ˆØ” Ñ&¨$¬+°gÔ*>×*@Ò*@Ñ*BÔ*BÑBÐBr   c                 ó@   — | j         j                             |¦  «        S r	   r2   r4   s     r   r5   z!AbsoluteDiscounting.unigram_scoreJ   r6   r   )r@   r7   r=   s   @r   r?   r?   7   ss   ø€ € € € € Ø+Ð+ð!ð !ð !ð !ð !ð !ðð ð ðCð Cð Cð/ð /ð /ð /ð /ð /ð /r   r?   c                   óJ   ‡ — e Zd ZdZdˆ fd„	Zd„ Zd„ Z e¦   «         fd„Zˆ xZ	S )Ú	KneserNeyañ  Kneser-Ney Smoothing.

    This is an extension of smoothing with a discount.

    Resources:
    - https://pages.ucsd.edu/~rlevy/lign256/winter2008/kneser_ney_mini_example.pdf
    - https://www.youtube.com/watch?v=ody1ysUTD7o
    - https://medium.com/@dennyc/a-simple-numerical-example-for-kneser-ney-smoothing-nlp-4600addf38b8
    - https://www.cl.uni-heidelberg.de/courses/ss15/smt/scribe6.pdf
    - https://www-i6.informatik.rwth-aachen.de/publications/download/951/Kneser-ICASSP-1995.pdf
    çš™™™™™¹?c                 óZ   •—  t          ¦   «         j        ||fi |¤Ž || _        || _        d S r	   )r   r   rB   Ú_order)r   r    r!   ÚorderrB   r"   r#   s         €r   r   zKneserNey.__init__[   s6   ø€ Ø�‰ŒÔ˜ WÐ7Ð7°Ð7Ð7Ð7Ø ˆŒØˆŒˆˆr   c                 ó<   — |                       |¦  «        \  }}||z  S r	   )Ú_continuation_counts)r   r)   Úword_continuation_countÚtotal_counts       r   r5   zKneserNey.unigram_score`   s&   € Ø/3×/HÒ/HÈÑ/NÔ/NÑ,Ð Ø&¨Ñ4Ð4r   c                 ó.  — | j         |         }t          |¦  «        dz   | j        k    r||         |                     ¦   «         fn|                      ||¦  «        \  }}t          || j        z
  d¦  «        |z  }| j        t          |¦  «        z  |z  }||fS )Nr   g        )r%   ÚlenrK   r   rN   rD   rB   r   )r   r)   r*   Úprefix_countsrO   rP   r+   r,   s           r   r-   zKneserNey.alpha_gammad   s£   € Øœ GÔ,ˆõ �7‰|Œ|˜aÑ 4¤;Ò.Ð.ð ˜4Ô  -§/¢/Ñ"3Ô"3Ð4Ð4à×*Ò*¨4°Ñ9Ô9ñ 	-Ð õ
 Ð+¨d¬mÑ;¸SÑAÔAÀKÑOˆØ”Õ 5°mÑ DÔ DÑDÀ{ÑRˆØ�eˆ|Ðr   c                 óò   ‡— ˆfd„| j         t          ‰¦  «        dz                                 ¦   «         D ¦   «         }d\  }}|D ]0}|t          ||         dk    ¦  «        z  }|t	          |¦  «        z  }Œ1||fS )a  Count continuations that end with context and word.

        Continuations track unique ngram "types", regardless of how many
        instances were observed for each "type".
        This is different than raw ngram counts which track number of instances.
        c              3   ó>   •K  — | ]\  }}|d d…         ‰k    ¯|V — ŒdS )r   Nr
   )r   Úprefix_ngramr%   r*   s      €r   r   z1KneserNey._continuation_counts.<locals>.<genexpr>v   sG   øè è € ð ,
ð ,
á$�˜fØ˜A˜B˜BÔ 7Ò*Ð*ð à*Ð*Ð*Ð*ð,
ð ,
r   é   )r   r   r   )r%   rR   ÚitemsÚintr   )r   r)   r*   Ú higher_order_ngrams_with_contextÚ#higher_order_ngrams_with_word_countÚtotalr%   s     `    r   rN   zKneserNey._continuation_countso   s¢   ø€ ð,
ð ,
ð ,
ð ,
à(,¬µC¸±L´LÀ1Ñ4DÔ(E×(KÒ(KÑ(MÔ(Mð,
ñ ,
ô ,
Ð(ð
 6:Ñ2Ð+¨UØ6ð 	3ð 	3ˆFØ/µ3°v¸d´|ÀaÒ7GÑ3HÔ3HÑHÐ/ØÕ*¨6Ñ2Ô2Ñ2ˆEˆEØ2°EÐ9Ð9r   )rI   )
r8   r9   r:   r;   r   r5   r-   ÚtuplerN   r<   r=   s   @r   rH   rH   N   s„   ø€ € € € € ð
ð 
ðð ð ð ð ð ð
5ð 5ð 5ð	ð 	ð 	ð 27°±´ð :ð :ð :ð :ð :ð :ð :ð :r   rH   N)r;   Úoperatorr   Únltk.lm.apir   Únltk.probabilityr   r   r   r?   rH   r
   r   r   ú<module>ra      sÞ   ððð ð
 "Ð !Ð !Ð !Ð !Ð !à !Ð !Ð !Ð !Ð !Ð !Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0ðð ð ð"/ð /ð /ð /ð /�ñ /ô /ð /ð&/ð /ð /ð /ð /˜)ñ /ô /ð /ð.1:ð 1:ð 1:ð 1:ð 1:�	ñ 1:ô 1:ð 1:ð 1:ð 1:r   