§
    'ê[f“  ã                   óŒ   — d Z ddlmZ ddlmZ 	 ddlZn# e$ r dZY nw xY w G d„ d¦  «        Zd„ Z	e
dk    r e	¦   «          dS dS )	a˜  
A module for language identification using the TextCat algorithm.
An implementation of the text categorization algorithm
presented in Cavnar, W. B. and J. M. Trenkle,
"N-Gram-Based Text Categorization".

The algorithm takes advantage of Zipf's law and uses
n-gram frequencies to profile languages and text-yet to
be identified-then compares using a distance measure.

Language n-grams are provided by the "An Crubadan"
project. A corpus reader was created separately to read
those files.

For details regarding the algorithm, see:
https://www.let.rug.nl/~vannoord/TextCat/textcat.pdf

For details about An Crubadan, see:
https://borel.slu.edu/crubadan/index.html
é    )Úmaxsize)ÚtrigramsNc                   óF   — e Zd ZdZi ZdZdZi Zd„ Zd„ Z	d„ Z
d„ Zd„ Zd	„ ZdS )
ÚTextCatNÚ<Ú>c                 ó¸   — t           st          d¦  «        ‚ddlm} || _        | j                             ¦   «         D ]}| j                             |¦  «         Œd S )Nz›classify.textcat requires the regex module that supports unicode. Try '$ pip install regex' and see https://pypi.python.org/pypi/regex for further details.r   )Úcrubadan)ÚreÚOSErrorÚnltk.corpusr
   Ú_corpusÚlangsÚ	lang_freq)Úselfr
   Úlangs      úI/var/www/piapp/venv/lib/python3.11/site-packages/nltk/classify/textcat.pyÚ__init__zTextCat.__init__8   s|   € Ýð 	Ýð#ñô ð ð 	)Ð(Ð(Ð(Ð(Ð(àˆŒà”L×&Ò&Ñ(Ô(ð 	)ð 	)ˆDØŒL×"Ò" 4Ñ(Ô(Ð(Ð(ð	)ð 	)ó    c                 ó.   — t          j        dd|¦  «        S )z)Get rid of punctuation except apostrophesz[^\P{P}\']+Ú )r   Úsub©r   Útexts     r   Úremove_punctuationzTextCat.remove_punctuationH   s   € åŒv�n b¨$Ñ/Ô/Ð/r   c                 ó  — ddl m}m} |                      |¦  «        } ||¦  «        } |¦   «         }|D ]L}t	          | j        |z   | j        z   ¦  «        }d„ |D ¦   «         }	|	D ]}
|
|v r||
xx         dz  cc<   Œd||
<   ŒŒM|S )z'Create FreqDist of trigrams within textr   )ÚFreqDistÚword_tokenizec                 ó8   — g | ]}d                       |¦  «        ‘ŒS )r   )Újoin)Ú.0Útris     r   ú
<listcomp>z#TextCat.profile.<locals>.<listcomp>V   s"   € ÐKÐKÐK¨s˜bŸgšg c™lœlÐKÐKÐKr   é   )Únltkr   r   r   r   Ú_START_CHARÚ	_END_CHAR)r   r   r   r   Ú
clean_textÚtokensÚfingerprintÚtÚtoken_trigram_tuplesÚtoken_trigramsÚcur_trigrams              r   ÚprofilezTextCat.profileL   sÚ   € à0Ð0Ð0Ð0Ð0Ð0Ð0Ð0à×,Ò,¨TÑ2Ô2ˆ
Ø�˜zÑ*Ô*ˆà�h‘j”jˆØð 	1ð 	1ˆAÝ#+¨DÔ,<¸qÑ,@À4Ä>Ñ,QÑ#RÔ#RÐ ØKÐKÐ6JÐKÑKÔKˆNà-ð 1ð 1�Ø +Ð-Ð-Ø Ð,Ð,Ô,°Ñ1Ð,Ð,Ñ,Ð,à/0�K Ñ,Ð,ð	1ð Ðr   c                 óJ  — | j                              |¦  «        }d}||v r{t          |                     ¦   «         ¦  «                             |¦  «        }t          |                     ¦   «         ¦  «                             |¦  «        }t          ||z
  ¦  «        }nt          }|S )zgCalculate the "out-of-place" measure between the
        text and language profile for a single trigramr   )r   r   ÚlistÚkeysÚindexÚabsr   )r   r   ÚtrigramÚtext_profileÚlang_fdÚdistÚidx_lang_profileÚidx_texts           r   Ú	calc_distzTextCat.calc_dist`   s—   € ð ”,×(Ò(¨Ñ.Ô.ˆØˆà�gÐÐÝ# G§L¢L¡N¤NÑ3Ô3×9Ò9¸'ÑBÔBÐÝ˜L×-Ò-Ñ/Ô/Ñ0Ô0×6Ò6°wÑ?Ô?ˆHõ Ð'¨(Ñ2Ñ3Ô3ˆDˆDõ
 ˆDàˆr   c                 óÂ   — i }|                       |¦  «        }| j        j                             ¦   «         D ](}d}|D ]}||                      |||¦  «        z  }Œ|||<   Œ)|S )zOCalculate the "out-of-place" measure between
        the text and all languagesr   )r/   r   Ú_all_lang_freqr2   r;   )r   r   Ú	distancesr/   r   Ú	lang_distr5   s          r   Ú
lang_distszTextCat.lang_distsu   s   € ð ˆ	Ø—,’,˜tÑ$Ô$ˆà”LÔ/×4Ò4Ñ6Ô6ð 	(ð 	(ˆDð ˆIØ"ð Dð D�Ø˜TŸ^š^¨D°'¸7ÑCÔCÑC�	�	à'ˆI�d‰OˆOàÐr   c                 óv   — |                       |¦  «        | _        t          | j        | j        j        ¬¦  «        S )zYFind the language with the min distance
        to the text and return its ISO 639-3 code)Úkey)r@   Úlast_distancesÚminÚgetr   s     r   Úguess_languagezTextCat.guess_language‡   s4   € ð #Ÿošo¨dÑ3Ô3ˆÔå�4Ô&¨DÔ,?Ô,CÐDÑDÔDÐDr   )Ú__name__Ú
__module__Ú__qualname__r   Úfingerprintsr&   r'   rC   r   r   r/   r;   r@   rF   © r   r   r   r   /   sŒ   € € € € € à€GØ€LØ€KØ€Ià€Nð)ð )ð )ð 0ð 0ð 0ðð ð ð(ð ð ð*ð ð ð$Eð Eð Eð Eð Er   r   c            
      ó6  — ddl m}  g d¢}dddddd	d
dddœ	}t          ¦   «         }|D ]ñ}|                      |¦  «        }t	          |¦  «        dz
  }t          t          t          |¦  «        ¦  «        }d}t          d|¦  «        D ]6}	d}
t          d||	         ¦  «        D ]}|
d||	         |         z   z  }
Œ||
z  }Œ7t          d|dd…         z   dz   ¦  «         | 	                    |¦  «        }t          d|› d||         › d�¦  «         t          d¦  «         Œòd S )Nr   )Úudhr)	zKurdish-UTF8zAbkhaz-UTF8zFarsi_Persian-UTF8z
Hindi-UTF8zHawaiian-UTF8zRussian-UTF8zVietnamese-UTF8zSerbian_Srpski-UTF8zEsperanto-UTF8zNorthern KurdishÚ	AbkhazianzIranian PersianÚHindiÚHawaiianÚRussianÚ
VietnameseÚSerbianÚ	Esperanto)	ÚkmrÚabkÚpesÚhinÚhawÚrusÚvieÚsrpÚepor$   r   Ú zLanguage snippet: éŒ   z...zLanguage detection: z (ú)zŒ############################################################################################################################################)
r   rM   r   ÚsentsÚlenr1   ÚmapÚrangeÚprintrF   )rM   r   ÚfriendlyÚtcÚcur_langÚraw_sentencesÚrowsÚcolsÚsampleÚiÚcur_sentÚjÚguesss                r   Údemorq   �   s€  € Ø Ð Ð Ð Ð Ð ð
ð 
ð 
€Eð "ØØ ØØØØØØð
ð 
€Hõ 
‰Œ€Bàð ð ˆàŸ
š
 8Ñ,Ô,ˆÝ�=Ñ!Ô! AÑ%ˆÝ•C�˜]Ñ+Ô+Ñ,Ô,ˆàˆõ �q˜$‘”ð 	ð 	ˆAØˆHÝ˜1˜d 1œgÑ&Ô&ð 6ð 6�Ø˜C -°Ô"2°1Ô"5Ñ5Ñ5��à�hÑˆFˆFõ 	Ð" V¨A¨c¨E¤]Ñ2°UÑ:Ñ;Ô;Ð;Ø×!Ò! &Ñ)Ô)ˆÝÐ@ UÐ@Ð@¨h°u¬oÐ@Ð@Ð@ÑAÔAÐAÝˆiÑÔÐÐð)ð r   Ú__main__)Ú__doc__Úsysr   Ú	nltk.utilr   Úregexr   ÚImportErrorr   rq   rG   rK   r   r   ú<module>rx      sÚ   ððð ð* Ð Ð Ð Ð Ð à Ð Ð Ð Ð Ð ðØÐÐÐÐøØð ð ð Ø	€B€B€Bðøøøð]Eð ]Eð ]Eð ]Eð ]Eñ ]Eô ]Eð ]EðB1ð 1ð 1ðh ˆzÒÐØ€D�F„F€F€F€Fð Ðs   � •ž