§
    �zIf(  ã                   óZ   — d Z ddlZddlZddlmZ ddlmZ ddlmZ  G d„ de¦  «        Z	dS )a  
This is a NLTK port of the tokenizer used in the NIST BLEU evaluation script,
https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/mteval-v14.pl#L926
which was also ported into Python in
https://github.com/lium-lst/nmtpy/blob/master/nmtpy/metrics/mtevalbleu.py#L162
é    N)Úperluniprops)Ú
TokenizerI)Úxml_unescapec            	       óx  — e Zd ZdZ ej        d¦  «        dfZ ej        d¦  «        dfZ ej        d¦  «        dfZ ej        d¦  «        d	fZ	 ej        d
¦  «        dfZ
 ej        d¦  «        d	fZee	e
egZ ed                      e ej        d¦  «        ¦  «        ¦  «        ¦  «        Z ed                      e ej        d¦  «        ¦  «        ¦  «        ¦  «        Z ed                      e ej        d¦  «        ¦  «        ¦  «        ¦  «        Z ej        dde¦  «        Z ej        dde¦  «        Z ej        dde¦  «        Z ej        d¦  «        dfZ ej        de› de› d�¦  «        d	fZ ej        de› de› d�¦  «        dfZ ej        de› d�¦  «        dfZeeeegZd„ Zdd„Z	 dd„Z dS )ÚNISTTokenizeruT  
    This NIST tokenizer is sentence-based instead of the original
    paragraph-based tokenization from mteval-14.pl; The sentence-based
    tokenization is consistent with the other tokenizers available in NLTK.

    >>> from nltk.tokenize.nist import NISTTokenizer
    >>> nist = NISTTokenizer()
    >>> s = "Good muffins cost $3.88 in New York."
    >>> expected_lower = [u'good', u'muffins', u'cost', u'$', u'3.88', u'in', u'new', u'york', u'.']
    >>> expected_cased = [u'Good', u'muffins', u'cost', u'$', u'3.88', u'in', u'New', u'York', u'.']
    >>> nist.tokenize(s, lowercase=False) == expected_cased
    True
    >>> nist.tokenize(s, lowercase=True) == expected_lower  # Lowercased.
    True

    The international_tokenize() is the preferred function when tokenizing
    non-european text, e.g.

    >>> from nltk.tokenize.nist import NISTTokenizer
    >>> nist = NISTTokenizer()

    # Input strings.
    >>> albb = u'Alibaba Group Holding Limited (Chinese: é˜¿é‡Œå·´å·´é›†å›¢æŽ§è‚¡ æœ‰é™�å…¬å�¸) us a Chinese e-commerce company...'
    >>> amz = u'Amazon.com, Inc. (/ËˆÃ¦mÉ™zÉ’n/) is an American electronic commerce...'
    >>> rkt = u'Rakuten, Inc. (æ¥½å¤©æ ªå¼�ä¼šç¤¾ Rakuten Kabushiki-gaisha) is a Japanese electronic commerce and Internet company based in Tokyo.'

    # Expected tokens.
    >>> expected_albb = [u'Alibaba', u'Group', u'Holding', u'Limited', u'(', u'Chinese', u':', u'é˜¿é‡Œå·´å·´é›†å›¢æŽ§è‚¡', u'æœ‰é™�å…¬å�¸', u')']
    >>> expected_amz = [u'Amazon', u'.', u'com', u',', u'Inc', u'.', u'(', u'/', u'ËˆÃ¦', u'm']
    >>> expected_rkt = [u'Rakuten', u',', u'Inc', u'.', u'(', u'æ¥½å¤©æ ªå¼�ä¼šç¤¾', u'Rakuten', u'Kabushiki', u'-', u'gaisha']

    >>> nist.international_tokenize(albb)[:10] == expected_albb
    True
    >>> nist.international_tokenize(amz)[:10] == expected_amz
    True
    >>> nist.international_tokenize(rkt)[:10] == expected_rkt
    True

    # Doctest for patching issue #1926
    >>> sent = u'this is a fooâ˜„sentence.'
    >>> expected_sent = [u'this', u'is', u'a', u'foo', u'â˜„', u'sentence', u'.']
    >>> nist.international_tokenize(sent) == expected_sent
    True
    z	<skipped>Ú u   â€¨ú z([\{-\~\[-\` -\&\(-\+\:-\@\/])z \1 z([^0-9])([\.,])z\1 \2 z([\.,])([^0-9])z \1 \2z
([0-9])(-)ÚNumberÚPunctuationÚSymbolz[]^\\-]z\\\g<0>z([ -]+)z([z])([z])c                 ó¤   — | j         \  }}|                     ||¦  «        }t          |¦  «        }| j        \  }}|                     ||¦  «        }|S )z8Performs the language independent string substituitions.)Ú
STRIP_SKIPÚsubr   ÚSTRIP_EOL_HYPHEN)ÚselfÚtextÚregexpÚsubstitutions       úF/var/www/piapp/venv/lib/python3.11/site-packages/nltk/tokenize/nist.pyÚlang_independent_subz"NISTTokenizer.lang_independent_sub   sU   € ð
  $œÑˆ�Ø�zŠz˜,¨Ñ-Ô-ˆÝ˜DÑ!Ô!ˆØ#Ô4Ñˆ�Ø�zŠz˜,¨Ñ-Ô-ˆØˆó    FTc                 ó�  — t          |¦  «        }|                      |¦  «        }|rAd|z   dz   }|r|                     ¦   «         }| j        D ]\  }}|                     ||¦  «        }Œd                     |                     ¦   «         ¦  «        }t          |                     ¦   «         ¦  «        }|r|n|                     ¦   «         S ©Nr	   )Ústrr   ÚlowerÚLANG_DEPENDENT_REGEXESr   ÚjoinÚsplitÚstrip)r   r   Ú	lowercaseÚwestern_langÚ
return_strr   r   s          r   ÚtokenizezNISTTokenizer.tokenize‹   s¾   € Ý�4‰yŒyˆà×(Ò(¨Ñ.Ô.ˆàð 	6à˜‘: Ñ#ˆDØð $Ø—z’z‘|”|�Ø(,Ô(Cð 6ð 6Ñ$�˜Ø—z’z ,°Ñ5Ô5��à�xŠx˜Ÿ
š
™œÑ%Ô%ˆõ �4—:’:‘<”<Ñ Ô ˆØ!Ð3ˆtˆt t§z¢z¡|¤|Ð3r   c                 óÒ  — t          |¦  «        }| j        \  }}|                     ||¦  «        }| j        \  }}|                     ||¦  «        }t	          |¦  «        }|r|                     ¦   «         }| j        D ]\  }}|                     ||¦  «        }Œd                     |                     ¦   «          	                    ¦   «         ¦  «        }|r|n| 	                    ¦   «         S r   )
r   r   r   r   r   r   ÚINTERNATIONAL_REGEXESr   r   r   )r   r   r    Úsplit_non_asciir"   r   r   s          r   Úinternational_tokenizez$NISTTokenizer.international_tokenizež   sÜ   € õ �4‰yŒyˆð  $œÑˆ�Ø�zŠz˜,¨Ñ-Ô-ˆØ#Ô4Ñˆ�Ø�zŠz˜,¨Ñ-Ô-ˆÝ˜DÑ!Ô!ˆàð 	 Ø—:’:‘<”<ˆDà$(Ô$>ð 	2ð 	2Ñ ˆF�LØ—:’:˜l¨DÑ1Ô1ˆDˆDð �xŠx˜Ÿ
š
™œ×*Ò*Ñ,Ô,Ñ-Ô-ˆØ!Ð3ˆtˆt t§z¢z¡|¤|Ð3r   N)FTF)!Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚreÚcompiler   r   ÚPUNCTÚPERIOD_COMMA_PRECEEDÚPERIOD_COMMA_FOLLOWÚDASH_PRECEED_DIGITr   r   r   Úsetr   ÚcharsÚ
pup_numberÚ	pup_punctÚ
pup_symbolr   Únumber_regexÚpunct_regexÚsymbol_regexÚNONASCIIÚPUNCT_1ÚPUNCT_2ÚSYMBOLSr%   r   r#   r'   © r   r   r   r      s†  € € € € € ð+ð +ð\ �”˜KÑ(Ô(¨"Ð,€Jà!�r”z (Ñ+Ô+¨SÐ0ÐàˆBŒJÐ8Ñ9Ô9¸7ÐB€Eà%˜2œ:Ð&8Ñ9Ô9¸:ÐEÐà$˜"œ*Ð%7Ñ8Ô8¸*ÐDÐà#˜œ LÑ1Ô1°:Ð=Ðð 	ØØØð	Ðð ��R—W’W˜S˜SÐ!3 Ô!3°HÑ!=Ô!=Ñ>Ô>Ñ?Ô?Ñ@Ô@€JØ��B—G’G˜C˜CÐ 2 Ô 2°=Ñ AÔ AÑBÔBÑCÔCÑDÔD€IØ��R—W’W˜S˜SÐ!3 Ô!3°HÑ!=Ô!=Ñ>Ô>Ñ?Ô?Ñ@Ô@€Jð �2”6˜* j°*Ñ=Ô=€LØ�"”&˜ Z°Ñ;Ô;€KØ�2”6˜* j°*Ñ=Ô=€Lð ˆrŒzÐ*Ñ+Ô+¨WÐ4€Hð 	ˆŒ
Ð9˜Ð9Ð9¨+Ð9Ð9Ð9Ñ:Ô:Øð€Gð
 	ˆŒ
Ð9˜Ð9Ð9¨Ð9Ð9Ð9Ñ:Ô:Øð€Gð
 ˆbŒjÐ.˜lÐ.Ð.Ð.Ñ/Ô/°Ð8€Gà% w°¸ÐAÐð
ð 
ð 
ð4ð 4ð 4ð 4ð( GLð4ð 4ð 4ð 4ð 4ð 4r   r   )
r+   Úior,   Únltk.corpusr   Únltk.tokenize.apir   Únltk.tokenize.utilr   r   r>   r   r   ú<module>rC      s—   ððð ð 
€	€	€	Ø 	€	€	€	à $Ð $Ð $Ð $Ð $Ð $Ø (Ð (Ð (Ð (Ð (Ð (Ø +Ð +Ð +Ð +Ð +Ð +ðY4ð Y4ð Y4ð Y4ð Y4�Jñ Y4ô Y4ð Y4ð Y4ð Y4r   