§
    �zIfÉx  ã                   ó´   — d Z ddlmZmZ ddlZddlmZmZmZm	Z	m
Z
mZmZmZ d„ Zej                              e¦   «          d¬¦  «        Z G d„ d	¦  «        ZdS )
zC
Unit tests for nltk.tokenize.
See also nltk/test/tokenize.doctest
é    )ÚListÚTupleN)ÚLegalitySyllableTokenizerÚStanfordSegmenterÚSyllableTokenizerÚTreebankWordTokenizerÚTweetTokenizerÚpunktÚsent_tokenizeÚword_tokenizec                  óš   — 	 t          ¦   «         } |                      d¦  «         |                      d¦  «         dS # t          $ r Y dS w xY w)NÚarÚzhTF)r   Údefault_configÚLookupError)Úsegs    úP/var/www/piapp/venv/lib/python3.11/site-packages/nltk/test/unit/test_tokenize.pyÚload_stanford_segmenterr      sa   € ðÝÑ!Ô!ˆØ×Ò˜4Ñ Ô Ð Ø×Ò˜4Ñ Ô Ð ØˆtøÝð ð ð Øˆuˆuðøøøs   ‚8< ¼
A
Á	A
z/NLTK was unable to find stanford-segmenter.jar.)Úreasonc                   óÊ  — e Zd Zd„ Zej                             ddg d¢g d¢ffdg d¢g d¢ffd	g d
¢g d
¢ffdg d¢g d¢ffdg d¢g d¢ffdg d¢g d¢ffdg d¢g d¢ffdg d¢g d¢ffg¦  «        dedee	e         e	e         f         fd„¦   «         Z
d„ Zd„ Zd„ Zed„ ¦   «         Zed „ ¦   «         Zd!„ Zd"„ Zd#„ Zd$„ Zd%„ Zd&„ Zd'„ Zd(„ Zd)„ Zd*„ Zd+„ Zd,„ Zej                             d-g d.¢¦  «        dNd0„¦   «         Zd1„ Zej                             d2d3g d4¢fd5g d6¢fd7d7gfd8d8gfd9d:d;gfd<d=d>gfd?g d@¢fdAdBd;gfdCdDd;gfdEg dF¢fdGg dF¢fdHdIdJgfg¦  «        dKedLe	e         fdM„¦   «         Zd/S )OÚTestTokenizec                 ón   — t          dd¬¦  «        }d}|                     |¦  «        }g d¢}||k    sJ ‚dS )zW
        Test TweetTokenizer using words with special and accented characters.
        T)Ústrip_handlesÚ
reduce_lenuA   @myke: Let's test these words: resumÃ© EspaÃ±a MÃ¼nchen franÃ§ais)
ú:zLet'sÚtestÚtheseÚwordsr   u   resumÃ©u   EspaÃ±au   MÃ¼nchenu	   franÃ§aisN©r	   Útokenize)ÚselfÚ	tokenizerÚs9ÚtokensÚexpecteds        r   Útest_tweet_tokenizerz!TestTokenize.test_tweet_tokenizer&   sX   € õ
 #°À$ÐGÑGÔGˆ	ØPˆØ×#Ò# BÑ'Ô'ˆð
ð 
ð 
ˆð ˜Ò!Ð!Ð!Ð!Ð!Ð!ó    ztest_input, expectedsz#My text 0106404243030 is great text)ÚMyÚtextÚ01064042430Ú30ÚisÚgreatr)   )r(   r)   Ú0106404243030r,   r-   r)   zMy ticket id is 1234543124123)r(   ÚticketÚidr,   Ú12345431241Ú23)r(   r/   r0   r,   Ú1234543124123z<@remy: This is waaaaayyyy too much for you!!!!!! 01064042430)r   ÚThisr,   ÚwaaayyyÚtooÚmuchÚforÚyouú!r:   r:   r*   z*My number is 06-46124080, except it's not.)	r(   Únumberr,   z06-46124080ú,Úexceptúit'sÚnotú.z+My number is 601-984-4813, except it's not.)	r(   r;   r,   z601-984-4813r<   r=   r>   r?   r@   )
r(   r;   r,   z601-984-Ú4813r<   r=   r>   r?   r@   z/My number is (393)  928 -3010, except it's not.)	r(   r;   r,   ú(393)  928 -3010r<   r=   r>   r?   r@   )r(   r;   r,   ú(Ú393ú)Ú928ú-Ú3010r<   r=   r>   r?   r@   z1The product identification number is 48103284512.)ÚTheÚproductÚidentificationr;   r,   Ú
4810328451Ú2r@   )rI   rJ   rK   r;   r,   Ú48103284512r@   z(My favourite substraction is 240 - 1353.)r(   Ú	favouriteÚsubstractionr,   z
240 - 1353r@   )r(   rO   rP   r,   Ú240rG   Ú1353r@   Ú
test_inputÚ	expectedsc                 ó”   — t          ddg|¦  «        D ]4\  }}t          dd|¬¦  «        }|                     |¦  «        }||k    sJ ‚Œ5dS )a¦  
        Test `match_phone_numbers` in TweetTokenizer.

        Note that TweetTokenizer is also passed the following for these tests:
            * strip_handles=True
            * reduce_len=True

        :param test_input: The input string to tokenize using TweetTokenizer.
        :type test_input: str
        :param expecteds: A 2-tuple of tokenized sentences. The first of the two
            tokenized is the expected output of tokenization with `match_phone_numbers=True`.
            The second of the two tokenized lists is the expected output of tokenization
            with `match_phone_numbers=False`.
        :type expecteds: Tuple[List[str], List[str]]
        TF)r   r   Úmatch_phone_numbersN)Úzipr	   r    )r!   rS   rT   rV   r%   r"   Ú	predicteds          r   Útest_tweet_tokenizer_expandedz*TestTokenize.test_tweet_tokenizer_expanded<   sx   € õH .1°$¸°À	Ñ-JÔ-Jð 	)ð 	)Ñ)Ð Ý&Ø"ØØ$7ðñ ô ˆIð
 "×*Ò*¨:Ñ6Ô6ˆIØ Ò(Ð(Ð(Ð(Ð(ð	)ð 	)r'   c                 ó`   — t          ¦   «         }|                     d¦  «        }|g d¢k    sJ ‚dS )ú3
        Test SyllableTokenizer tokenizer.
        Újustification)ÚjusÚtiÚfiÚcaÚtionN©r   r    )r!   r"   r$   s      r   Ú+test_sonority_sequencing_syllable_tokenizerz8TestTokenize.test_sonority_sequencing_syllable_tokenizer	  s@   € õ &Ñ'Ô'ˆ	Ø×#Ò# OÑ4Ô4ˆØÐ:Ð:Ð:Ò:Ð:Ð:Ð:Ð:Ð:r'   c                 óh   — t          ¦   «         }ddz  }|                     |¦  «        }||gk    sJ ‚dS )r[   Ú9i'  Nrb   )r!   r"   r)   r$   s       r   Útest_syllable_tokenizer_numbersz,TestTokenize.test_syllable_tokenizer_numbers  sD   € õ &Ñ'Ô'ˆ	Ø�U‰{ˆØ×#Ò# DÑ)Ô)ˆØ˜$˜ÒÐÐÐÐÐr'   c                 ó–   — ddl m} d}t          |                     ¦   «         ¦  «        }|                     |¦  «        }|g d¢k    sJ ‚dS )z;
        Test LegalitySyllableTokenizer tokenizer.
        r   )r   Ú	wonderful)ÚwonÚderÚfulN)Únltk.corpusr   r   r    )r!   r   Ú	test_wordr"   r$   s        r   Ú*test_legality_principle_syllable_tokenizerz7TestTokenize.test_legality_principle_syllable_tokenizer  sa   € ð 	&Ð%Ð%Ð%Ð%Ð%àˆ	Ý-¨e¯kªk©m¬mÑ<Ô<ˆ	Ø×#Ò# IÑ.Ô.ˆØÐ.Ð.Ð.Ò.Ð.Ð.Ð.Ð.Ð.r'   c                 óÖ   — t          ¦   «         }|                     d¦  «         d}|                     |                     ¦   «         ¦  «        }|                     ¦   «         g d¢k    sJ ‚dS )zN
        Test the Stanford Word Segmenter for Arabic (default config)
        r   un   ÙŠØ¨Ø­Ø« Ø¹Ù„Ù… Ø§Ù„Ø­Ø§Ø³ÙˆØ¨ Ø§Ø³ØªØ®Ø¯Ø§Ù… Ø§Ù„Ø­ÙˆØ³Ø¨Ø© Ø¨Ø¬Ù…ÙŠØ¹ Ø§Ø´ÙƒØ§Ù„Ù‡Ø§ Ù„Ø­Ù„ Ø§Ù„Ù…Ø´ÙƒÙ„Ø§Øª)u   ÙŠØ¨Ø­Ø«u   Ø¹Ù„Ù…u   Ø§Ù„Ø­Ø§Ø³ÙˆØ¨u   Ø§Ø³ØªØ®Ø¯Ø§Ù…u   Ø§Ù„Ø­ÙˆØ³Ø¨Ø©u   Ø¨u   Ø¬Ù…ÙŠØ¹u
   Ø§Ø´ÙƒØ§Ù„u   Ù‡Ø§u   Ù„u   Ø­Ù„u   Ø§Ù„Ù…Ø´ÙƒÙ„Ø§ØªN©r   r   ÚsegmentÚsplit©r!   r   ÚsentÚsegmented_sents       r   Útest_stanford_segmenter_arabicz+TestTokenize.test_stanford_segmenter_arabic%  s}   € õ
  Ñ!Ô!ˆØ×Ò˜4Ñ Ô Ð ØˆØŸš T§Z¢Z¡\¤\Ñ2Ô2ˆØ×#Ò#Ñ%Ô%ð *
ð *
ð *
ò 
ð 
ð 
ð 
ð 
ð 
r'   c                 óÖ   — t          ¦   «         }|                     d¦  «         d}|                     |                     ¦   «         ¦  «        }|                     ¦   «         g d¢k    sJ ‚dS )zO
        Test the Stanford Word Segmenter for Chinese (default config)
        r   u$   è¿™æ˜¯æ–¯å�¦ç¦�ä¸­æ–‡åˆ†è¯�å™¨æµ‹è¯•)u   è¿™u   æ˜¯u	   æ–¯å�¦ç¦�u   ä¸­æ–‡u	   åˆ†è¯�å™¨u   æµ‹è¯•Nrp   rs   s       r   Útest_stanford_segmenter_chinesez,TestTokenize.test_stanford_segmenter_chinese=  sk   € õ
  Ñ!Ô!ˆØ×Ò˜4Ñ Ô Ð Ø5ˆØŸš T§Z¢Z¡\¤\Ñ2Ô2ˆØ×#Ò#Ñ%Ô%Ð)eÐ)eÐ)eÒeÐeÐeÐeÐeÐer'   c                 ó¬   — t          ¦   «         }d}dg}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚dS )zT
        Test a string that resembles a phone number but contains a newline
        rB   z(393)
928 -3010)rC   rD   rE   z	928 -3010Nr   )r!   r"   Útest1r%   ÚresultÚtest2s         r   Útest_phone_tokenizerz!TestTokenize.test_phone_tokenizerH  s|   € õ #Ñ$Ô$ˆ	Ø"ˆØ&Ð'ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð #ˆØ1Ð1Ð1ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!Ð!Ð!r'   c                 óÂ  — t          ¦   «         }d}dg}|                     |¦  «        }||k    sJ ‚d}dg}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d	}	g d
¢}|                     |	¦  «        }||k    sJ ‚dS )zX
        Test a string that contains Emoji ZWJ Sequences and skin tone modifier
        u   ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘§â€�ðŸ‘§u   ðŸ‘¨ðŸ�¿uŒ   ðŸ¤” ðŸ™ˆ me asÃ­, seðŸ˜Œ ds ðŸ’•ðŸ‘­ðŸ‘™ hello ðŸ‘©ðŸ�¾â€�ðŸŽ“ emoji hello ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘¦â€�ðŸ‘¦ how are ðŸ˜Š you todayðŸ™…ðŸ�½ðŸ™…ðŸ�½)u   ðŸ¤”u   ðŸ™ˆÚmeu   asÃ­r<   Úseu   ðŸ˜ŒÚdsu   ðŸ’•u   ðŸ‘­u   ðŸ‘™Úhellou   ðŸ‘©ðŸ�¾â€�ðŸŽ“Úemojir‚   u   ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘¦â€�ðŸ‘¦ÚhowÚareu   ðŸ˜Šr9   Útodayõ   ðŸ™…ðŸ�½r‡   u   ðŸ‡¦ðŸ‡µðŸ‡µðŸ‡±ðŸ‡ª)u   ðŸ‡¦ðŸ‡µõ   ðŸ‡µðŸ‡±u   ðŸ‡ªu   Hi ðŸ‡¨ðŸ‡¦, ðŸ˜�!!)ÚHiõ   ðŸ‡¨ðŸ‡¦r<   u   ðŸ˜�r:   r:   u   <3 ðŸ‡¨ðŸ‡¦ ðŸ¤� ðŸ‡µðŸ‡± <3)ú<3rŠ   u   ðŸ¤�rˆ   r‹   Nr   )
r!   r"   rz   r%   r{   r|   Útest3Útest4Útest5Útest6s
             r   Útest_emoji_tokenizerz!TestTokenize.test_emoji_tokenizer[  sW  € õ #Ñ$Ô$ˆ	ð ,ˆØ/Ð0ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð ˆØ�<ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð _ˆð
ð 
ð 
ˆð2 ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð 'ˆØ3Ð3Ð3ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!à%ˆØ<Ð<Ð<ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!à.ˆØ?Ð?Ð?ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!Ð!Ð!r'   c                 ó<   — d}g d¢}t          |¦  «        |k    sJ ‚dS )zA
        Test padding of asterisk for word tokenization.
        z1This is a, *weird sentence with *asterisks in it.)r4   r,   Úar<   Ú*ÚweirdÚsentenceÚwithr“   Ú	asterisksÚinÚitr@   N©r   ©r!   r)   r%   s      r   Útest_pad_asteriskzTestTokenize.test_pad_asteriskœ  s<   € ð Cˆð
ð 
ð 
ˆõ ˜TÑ"Ô" hÒ.Ð.Ð.Ð.Ð.Ð.r'   c                 ó<   — d}g d¢}t          |¦  «        |k    sJ ‚dS )z@
        Test padding of dotdot* for word tokenization.
        zPWhy did dotdot.. not get tokenized but dotdotdot... did? How about manydots.....)ÚWhyÚdidÚdotdotz..r?   ÚgetÚ	tokenizedÚbutÚ	dotdotdotz...rŸ   ú?ÚHowÚaboutÚmanydotsz.....Nrš   r›   s      r   Útest_pad_dotdotzTestTokenize.test_pad_dotdot²  s<   € ð bˆð
ð 
ð 
ˆõ$ ˜TÑ"Ô" hÒ.Ð.Ð.Ð.Ð.Ð.r'   c                 ó  — t          d¬¦  «        }d}g d¢}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d	}g d
¢}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d}	g d¢}|                     |	¦  «        }||k    sJ ‚d}
g d¢}|                     |
¦  «        }||k    sJ ‚dS )zW
        Test remove_handle() from casual.py with specially crafted edge cases
        T)r   z-@twitter hello @twi_tter_. hi @12345 @123news)r‚   r@   Úhiu]   @n`@n~@n(@n)@n-@n=@n+@n\@n|@n[@n]@n{@n}@n;@n:@n'@n"@n/@n?@n.@n,@n<@n>@n @n
@n Ã±@n.Ã¼@n.Ã§@n.)ú`ú~rC   rE   rG   ú=ú+ú\ú|ú[ú]Ú{Ú}ú;r   ú'ú"ú/r¥   r@   r<   ú<ú>õ   Ã±r@   õ   Ã¼r@   õ   Ã§r@   zKa@n j@n z@n A@n L@n Z@n 1@n 4@n 7@n 9@n 0@n _@n !@n @@n #@n $@n %@n &@n *@n)&r’   ú@nÚjr¿   Úzr¿   ÚAr¿   ÚLr¿   ÚZr¿   Ú1r¿   Ú4r¿   Ú7r¿   re   r¿   Ú0r¿   Ú_r¿   r:   r¿   ú@r¿   ú#r¿   ú$r¿   ú%r¿   ú&r¿   r“   r¿   z@n!a @n#a @n$a @n%a @n&a @n*a)r:   r’   rË   r’   rÌ   r’   rÍ   r’   rÎ   r’   r“   r’   zD@n!@n @n#@n @n$@n @n%@n @n&@n @n*@n @n@n @@n @n@@n @n_@n @n7@n @nj@n)r:   r¿   rË   r¿   rÌ   r¿   rÍ   r¿   rÎ   r¿   r“   r¿   r¿   r¿   rÊ   r¿   r¿   rÊ   r¿   z@n_r¿   z@n7r¿   z@njr¿   z^@abcdefghijklmnopqrstuvwxyz @abcdefghijklmno1234 @abcdefghijklmno_ @abcdefghijklmnoendofhandle)ÚpqrstuvwxyzÚ1234rÉ   Úendofhandlez^@abcdefghijklmnop@abcde @abcdefghijklmno@abcde @abcdefghijklmno_@abcde @abcdefghijklmno5@abcde)Úpú@abcdez@abcdefghijklmnorÓ   rÉ   rÓ   Ú5rÓ   Nr   )r!   r"   rz   r%   r{   r|   rŒ   r�   rŽ   r�   Útest7s              r   Útest_remove_handlezTestTokenize.test_remove_handleË  s»  € õ
 #°Ð6Ñ6Ô6ˆ	ð @ˆØ'Ð'Ð'ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð sˆð
ð 
ð 
ˆð> ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð ^ˆð'
ð '
ð '
ˆðP ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð 0ˆØOÐOÐOˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð Wˆð
ð 
ð 
ˆð6 ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð qˆØ>Ð>Ð>ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð qˆð	
ð 	
ð 	
ˆð ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!Ð!Ð!r'   c                 óB  — t          ¦   «         }d}g d¢}t          |                     |¦  «        ¦  «        }||k    sJ ‚d}g d¢}t          |                     |¦  «        ¦  «        }||k    sJ ‚d}g d¢}t          |                     |¦  «        ¦  «        }||k    sJ ‚dS )zC
        Test TreebankWordTokenizer.span_tokenize function
        zNGood muffins cost $3.88
in New (York).  Please (buy) me
two of them.
(Thanks).))r   é   )é   é   )é   é   )é   é   )rÞ   é   )é   é   )é   é   )é   é    )rå   é$   )ræ   é%   )rç   é&   )é(   é.   )é/   é0   )rì   é3   )rí   é4   )é5   é7   )é8   é;   )é<   é>   )é?   éD   )éE   éF   )rø   éL   )rù   éM   )rú   éN   zmThe DUP is similar to the "religious right" in the United States and takes a hardline stance on social issues)©r   é   ©rØ   é   ©é   é
   ©é   rÝ   ©rÞ   é   ©é   é   ©rá   râ   ©râ   ræ   ©rç   é*   ©r  é+   ©é,   rê   ©rë   é2   ©rí   é9   ©é:   é@   ©éA   rö   ©r÷   éJ   ©éK   rù   )rú   éU   )éV   é\   )é]   é_   )é`   éf   )ég   ém   zqThe DUP is similar to the "religious right" in the United States and takes a ``hardline'' stance on social issues)rü   rþ   r   r  r  r  r
  r  r  r  r  r  r  r  r  r  r  )rú   éO   )r(  éW   )r)  éY   )éZ   r$  )éa   éc   )éd   éj   )ék   éq   N)r   ÚlistÚspan_tokenize)r!   r"   rz   r%   r{   r|   rŒ   s          r   Útest_treebank_span_tokenizerz)TestTokenize.test_treebank_span_tokenizerd  sê   € õ
 *Ñ+Ô+ˆ	ð dˆð
ð 
ð 
ˆõ2 �i×-Ò-¨eÑ4Ô4Ñ5Ô5ˆØ˜Ò!Ð!Ð!Ð!ð @ˆð
ð 
ð 
ˆõ0 �i×-Ò-¨eÑ4Ô4Ñ5Ô5ˆØ˜Ò!Ð!Ð!Ð!ð Fˆð
ð 
ð 
ˆõ4 �i×-Ò-¨eÑ4Ô4Ñ5Ô5ˆØ˜Ò!Ð!Ð!Ð!Ð!Ð!r'   c                 ór   — d}g d¢}t          |¦  «        |k    sJ ‚d}g d¢}t          |¦  «        |k    sJ ‚dS )z-
        Test word_tokenize function
        z0The 'v', I've been fooled but I'll seek revenge.)rI   r·   Úvr·   r<   ÚIz'veÚbeenÚfooledr£   r7  z'llÚseekÚrevenger@   z'v' 're')r·   r6  r·   z'rer·   Nrš   )r!   r•   r%   s      r   Útest_word_tokenizezTestTokenize.test_word_tokenizeÅ  se   € ð
 Fˆð
ð 
ð 
ˆõ" ˜XÑ&Ô&¨(Ò2Ð2Ð2Ð2àˆØ.Ð.Ð.ˆÝ˜XÑ&Ô&¨(Ò2Ð2Ð2Ð2Ð2Ð2r'   c                 ó„   — dddgfdg d¢fdg d¢fg}|D ]+\  }}d„ t          j        |¦  «        D ¦   «         }||k    sJ ‚Œ,d S )	NÚ12©rÅ   rM   )rM   NÚ123)r?  ©rM   Ú3)rB  NrÐ   )r?  rA  )rB  rÆ   )rÆ   Nc                 ó   — g | ]}|‘ŒS © rD  )Ú.0Úxs     r   ú
<listcomp>z5TestTokenize.test_punkt_pair_iter.<locals>.<listcomp>ë  s   € ÐEÐEÐE 1˜QÐEÐEÐEr'   )r
   Ú
_pair_iter)r!   Ú
test_casesrS   Úexpected_outputÚactual_outputs        r   Útest_punkt_pair_iterz!TestTokenize.test_punkt_pair_iterâ  sˆ   € ð �J Ð,Ð-ØÐ9Ð9Ð9Ð:ØÐFÐFÐFÐGð
ˆ
ð .8ð 	4ð 	4Ñ)ˆZ˜ØEÐE­Ô(8¸Ñ(DÔ(DÐEÑEÔEˆMà  OÒ3Ð3Ð3Ð3Ð3ð	4ð 	4r'   c                 ój   — t          g ¦  «        }t          j        |¦  «        }t          |¦  «         d S ©N)Úiterr
   rH  r2  )r!   r™   Úgens      r   Ú5test_punkt_pair_iter_handles_stop_iteration_exceptionzBTestTokenize.test_punkt_pair_iter_handles_stop_iteration_exceptionï  s,   € å�"‰XŒXˆåÔ˜rÑ"Ô"ˆåˆS‰	Œ	ˆ	ˆ	ˆ	r'   c                 ó¨   — t          j        ¦   «         } G d„ d¦  «        } |¦   «         |_        t          |                     d¦  «        ¦  «         d S )Nc                   ó   — e Zd Zd„ ZdS )úkTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exception.<locals>.TestPunktTokenizeWordsMockc                 ó    — t          g ¦  «        S rN  )rO  )r!   Úss     r   r   zyTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exception.<locals>.TestPunktTokenizeWordsMock.word_tokenizeû  s   € Ý˜B‘x”x�r'   N)Ú__name__Ú
__module__Ú__qualname__r   rD  r'   r   ÚTestPunktTokenizeWordsMockrT  ú  s#   € € € € € ð ð  ð  ð  ð  r'   rZ  r   )r
   ÚPunktBaseClassÚ
_lang_varsr2  Ú_tokenize_words)r!   ÚobjrZ  s      r   Ú:test_punkt_tokenize_words_handles_stop_iteration_exceptionzGTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exception÷  sl   € ÝÔ"Ñ$Ô$ˆð	 ð 	 ð 	 ð 	 ð 	 ñ 	 ô 	 ð 	 ð 4Ð3Ñ5Ô5ˆŒåˆS× Ò  Ñ(Ô(Ñ)Ô)Ð)Ð)Ð)r'   c                 ó²   —  G d„ dt           j        ¦  «        }t          j         |¦   «         ¬¦  «        }d}g d¢}|                     |¦  «        |k    sJ ‚d S )Nc                   ó   — e Zd ZdZdS )úNTestTokenize.test_punkt_tokenize_custom_lang_vars.<locals>.BengaliLanguageVars)r@   r¥   r:   u   à¥¤N©rW  rX  rY  Úsent_end_charsrD  r'   r   ÚBengaliLanguageVarsrb    s   € € € € € Ø6ˆNˆNˆNr'   re  )Ú	lang_varsõc  à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¶à§�à¦°à§€ à¦�à¦® à¦­à§‡à¦™à§�à¦•à¦¾à¦‡à§Ÿà¦¾ à¦¨à¦¾à¦‡à¦¡à§� à¦¸à§‹à¦®à¦¬à¦¾à¦° à¦†à¦‡ à¦†à¦‡ à¦Ÿà¦¿ à¦¦à¦¿à¦²à§�à¦²à¦¿à¦° à¦¹à§€à¦°à¦• à¦œà§Ÿà¦¨à§�à¦¤à§€ à¦‰à¦¦à¦¯à¦¾à¦ªà¦¨à§‡à¦° à¦‰à¦¦à§�à¦¬à§‹à¦§à¦¨ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤ à¦…à¦¨à¦²à¦¾à¦‡à¦¨à§‡à¦° à¦®à¦¾à¦§à§�à¦¯à¦®à§‡ à¦�à¦‡ à¦…à¦¨à§�à¦·à§�à¦ à¦¾à¦¨à§‡ à¦•à§‡à¦¨à§�à¦¦à§�à¦°à§€à§Ÿ à¦®à¦¾à¦¨à¦¬ à¦¸à¦®à§�à¦ªà¦¦ à¦‰à¦¨à§�à¦¨à§Ÿà¦¨à¦®à¦¨à§�à¦¤à§�à¦°à§€ à¦¶à§�à¦°à§€ à¦°à¦®à§‡à¦¶ à¦ªà§‹à¦–à¦°à¦¿à§Ÿà¦¾à¦² â€˜à¦¨à¦¿à¦¶à¦¾à¦™à§�à¦•â€™  à¦‰à¦ªà¦¸à§�à¦¥à¦¿à¦¤ à¦›à¦¿à¦²à§‡à¦¨à¥¤ à¦�à¦‡ à¦‰à¦ªà¦²à¦•à§�à¦·à§�à¦¯à§‡ à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¹à§€à¦°à¦•à¦œà§Ÿà¦¨à§�à¦¤à§€à¦° à¦²à§‹à¦—à§‹ à¦�à¦¬à¦‚ à§¨à§¦à§©à§¦-à¦�à¦° à¦œà¦¨à§�à¦¯ à¦ªà§�à¦°à¦¤à¦¿à¦·à§�à¦ à¦¾à¦¨à§‡à¦° à¦²à¦•à§�à¦·à§�à¦¯ à¦“ à¦ªà¦°à¦¿à¦•à¦²à§�à¦ªà¦¨à¦¾à¦° à¦¨à¦¥à¦¿ à¦ªà§�à¦°à¦•à¦¾à¦¶ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤)u  à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¶à§�à¦°à§€ à¦�à¦® à¦­à§‡à¦™à§�à¦•à¦¾à¦‡à§Ÿà¦¾ à¦¨à¦¾à¦‡à¦¡à§� à¦¸à§‹à¦®à¦¬à¦¾à¦° à¦†à¦‡ à¦†à¦‡ à¦Ÿà¦¿ à¦¦à¦¿à¦²à§�à¦²à¦¿à¦° à¦¹à§€à¦°à¦• à¦œà§Ÿà¦¨à§�à¦¤à§€ à¦‰à¦¦à¦¯à¦¾à¦ªà¦¨à§‡à¦° à¦‰à¦¦à§�à¦¬à§‹à¦§à¦¨ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤u+  à¦…à¦¨à¦²à¦¾à¦‡à¦¨à§‡à¦° à¦®à¦¾à¦§à§�à¦¯à¦®à§‡ à¦�à¦‡ à¦…à¦¨à§�à¦·à§�à¦ à¦¾à¦¨à§‡ à¦•à§‡à¦¨à§�à¦¦à§�à¦°à§€à§Ÿ à¦®à¦¾à¦¨à¦¬ à¦¸à¦®à§�à¦ªà¦¦ à¦‰à¦¨à§�à¦¨à§Ÿà¦¨à¦®à¦¨à§�à¦¤à§�à¦°à§€ à¦¶à§�à¦°à§€ à¦°à¦®à§‡à¦¶ à¦ªà§‹à¦–à¦°à¦¿à§Ÿà¦¾à¦² â€˜à¦¨à¦¿à¦¶à¦¾à¦™à§�à¦•â€™  à¦‰à¦ªà¦¸à§�à¦¥à¦¿à¦¤ à¦›à¦¿à¦²à§‡à¦¨à¥¤u/  à¦�à¦‡ à¦‰à¦ªà¦²à¦•à§�à¦·à§�à¦¯à§‡ à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¹à§€à¦°à¦•à¦œà§Ÿà¦¨à§�à¦¤à§€à¦° à¦²à§‹à¦—à§‹ à¦�à¦¬à¦‚ à§¨à§¦à§©à§¦-à¦�à¦° à¦œà¦¨à§�à¦¯ à¦ªà§�à¦°à¦¤à¦¿à¦·à§�à¦ à¦¾à¦¨à§‡à¦° à¦²à¦•à§�à¦·à§�à¦¯ à¦“ à¦ªà¦°à¦¿à¦•à¦²à§�à¦ªà¦¨à¦¾à¦° à¦¨à¦¥à¦¿ à¦ªà§�à¦°à¦•à¦¾à¦¶ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤)r
   ÚPunktLanguageVarsÚPunktSentenceTokenizerr    )r!   re  r^  Ú	sentencesr%   s        r   Ú$test_punkt_tokenize_custom_lang_varsz1TestTokenize.test_punkt_tokenize_custom_lang_vars  sŒ   € ð	7ð 	7ð 	7ð 	7ð 	7¥%Ô"9ñ 	7ô 	7ð 	7õ Ô*Ð5HÐ5HÑ5JÔ5JÐKÑKÔKˆð zˆ	ð
ð 
ð 
ˆð �|Š|˜IÑ&Ô&¨(Ò2Ð2Ð2Ð2Ð2Ð2r'   c                 ól   — t          j        ¦   «         }d}dg}|                     |¦  «        |k    sJ ‚d S )Nrg  )r
   ri  r    )r!   r^  rj  r%   s       r   Ú'test_punkt_tokenize_no_custom_lang_varsz4TestTokenize.test_punkt_tokenize_no_custom_lang_vars  sJ   € åÔ*Ñ,Ô,ˆð zˆ	ð rð
ˆð �|Š|˜IÑ&Ô&¨(Ò2Ð2Ð2Ð2Ð2Ð2r'   z%input_text,n_sents,n_splits,lang_vars))z4Subject: Some subject. Attachments: Some attachmentsé   é   )z4Subject: Some subject! Attachments: Some attachmentsrn  ro  )z4This is just a normal sentence, just like any other.ro  r   Nc                 ó   — t          j        ¦   «         }|d k    r||_        t          |                     |¦  «        ¦  «        |k    sJ ‚t          t          |                     |¦  «        ¦  «        ¦  «        |k    sJ ‚d S rN  )r
   ri  r\  Úlenr    r2  Údebug_decisions)r!   Ú
input_textÚn_sentsÚn_splitsrf  r"   s         r   Úpunkt_debug_decisionsz"TestTokenize.punkt_debug_decisions   s�   € õ$ Ô0Ñ2Ô2ˆ	Ø˜ÒÐØ#,ˆIÔ å�9×%Ò% jÑ1Ô1Ñ2Ô2°gÒ=Ð=Ð=Ð=Ý•4˜	×1Ò1°*Ñ=Ô=Ñ>Ô>Ñ?Ô?À8ÒKÐKÐKÐKÐKÐKr'   c                 óx   —  G d„ dt           j        ¦  «        }|                      ddd |¦   «         ¬¦  «         d S )Nc                   ó   — e Zd ZdZdS )úGTestTokenize.test_punkt_debug_decisions_custom_end.<locals>.ExtLangVars)r@   r¥   r:   ú^Nrc  rD  r'   r   ÚExtLangVarsry  <  s   € € € € € Ø1ˆNˆNˆNr'   r{  z4Subject: Some subject^ Attachments: Some attachmentsrn  ro  )rt  ru  rf  )r
   rh  rv  )r!   r{  s     r   Ú%test_punkt_debug_decisions_custom_endz2TestTokenize.test_punkt_debug_decisions_custom_end9  sj   € ð	2ð 	2ð 	2ð 	2ð 	2�%Ô1ñ 	2ô 	2ð 	2ð 	×"Ò"ØBØØØ!�k‘m”mð	 	#ñ 	
ô 	
ð 	
ð 	
ð 	
r'   zsentences, expectedzthis is a test. . new sentence.)zthis is a test.r@   znew sentence.zThis. . . That)úThis.r@   r@   ÚThatzThis..... ThatzThis... ThatzThis.. . ThatzThis.. .r~  zThis. .. Thatr}  z.. ThatzThis. ,. That)r}  z,.r~  zThis!!! ThatzThis!!!z
This! ThatzThis!z+1. This is R .
2. This is A .
3. That's all)z1.zThis is R .z2.zThis is A .z3.z
That's allz+1. This is R .	2. This is A .	3. That's allzHello.	TherezHello.ÚThererj  r%   c                 ó0   — t          |¦  «        |k    sJ ‚d S rN  )r   )r!   rj  r%   s      r   Útest_sent_tokenizezTestTokenize.test_sent_tokenizeH  s#   € õ6 ˜YÑ'Ô'¨8Ò3Ð3Ð3Ð3Ð3Ð3r'   rN  ) rW  rX  rY  r&   ÚpytestÚmarkÚparametrizeÚstrr   r   rY   rc   rf   rn   Úcheck_stanford_segmenterrv   rx   r}   r�   rœ   r©   rÖ   r4  r<  rL  rQ  r_  rk  rm  rv  r|  r�  rD  r'   r   r   r   %   sT  € € € € € ð"ð "ð "ð, „[×ÒØð 6àNÐNÐNØJÐJÐJððð 0àEÐEÐEØAÐAÐAððð Oðð ð ðð ð ðð ðL =ð
ð 
ð 
ð
ð 
ð 
ððð< >ð
ð 
ð 
ðð ð ððð> Bð
ð 
ð 
ðð ð ðð ðF Dð	ð 	ð 	ðð ð ððð6 ;àPÐPÐPØVÐVÐVðððMn	
ñqô qðd)Øð)Ø*/°°S´	¸4À¼9Ð0DÔ*Eð)ð )ð )ñeqô qðd)ð6;ð ;ð ;ð ð  ð  ð	/ð 	/ð 	/ð ð
ð 
ñ Ôð
ð. ðfð fñ Ôðfð"ð "ð "ð&?"ð ?"ð ?"ðB/ð /ð /ð,/ð /ð /ð2W"ð W"ð W"ðr_"ð _"ð _"ðB3ð 3ð 3ð:4ð 4ð 4ðð ð ð	*ð 	*ð 	*ð3ð 3ð 3ð$
3ð 
3ð 
3ð „[×ÒØ/ð	
ð 	
ð 	
ñô ð"Lð Lð Lñ#ô ð"Lð
ð 
ð 
ð „[×ÒØð 2Ø9Ð9Ð9ðð Ð:Ð:Ð:Ð;ØÐ 0Ð1Ð2Ø˜nÐ-Ð.Ø˜z¨6Ð2Ð3Ø˜w¨	Ð2Ð3ØÐ5Ð5Ð5Ð6Ø˜i¨Ð0Ð1Ø˜G VÐ,Ð-à?ØNÐNÐNðð
 @ØNÐNÐNðð ˜x¨Ð1Ð2ð+	
ñô ð44¨Cð 4¸4À¼9ð 4ð 4ð 4ñ5ô ð44ð 4ð 4r'   r   )Ú__doc__Útypingr   r   r‚  Únltk.tokenizer   r   r   r   r	   r
   r   r   r   rƒ  Úskipifr†  r   rD  r'   r   ú<module>r‹     s  ððð ð Ð Ð Ð Ð Ð Ð Ð à €€€ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ðð ð ð "œ;×-Ò-ØÐÑ!Ô!Ð!Ø<ð .ñ ô Ð ð~4ð ~4ð ~4ð ~4ð ~4ñ ~4ô ~4ð ~4ð ~4ð ~4r'   