§
    'ê[f�&  ã                   ó~   — d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZ d dlmZ dZ G d„ de¦  «        ZdS )é    N)ÚPIPE)Ú_java_optionsÚconfig_javaÚfind_dirÚ	find_fileÚfind_jarÚjava)Ú
TokenizerIz!https://nlp.stanford.edu/softwarec                   óf   ‡ — e Zd ZdZdZ	 	 	 	 	 	 	 	 	 	 	 	 dd„Zd	„ Zˆ fd
„Zd„ Zd„ Z	d„ Z
dd„Zˆ xZS )ÚStanfordSegmenteru[  Interface to the Stanford Segmenter

    If stanford-segmenter version is older than 2016-10-31, then path_to_slf4j
    should be provieded, for example::

        seg = StanfordSegmenter(path_to_slf4j='/YOUR_PATH/slf4j-api.jar')

    >>> from nltk.tokenize.stanford_segmenter import StanfordSegmenter
    >>> seg = StanfordSegmenter() # doctest: +SKIP
    >>> seg.default_config('zh') # doctest: +SKIP
    >>> sent = u'è¿™æ˜¯æ–¯å�¦ç¦�ä¸­æ–‡åˆ†è¯�å™¨æµ‹è¯•'
    >>> print(seg.segment(sent)) # doctest: +SKIP
    è¿™ æ˜¯ æ–¯å�¦ç¦� ä¸­æ–‡ åˆ†è¯�å™¨ æµ‹è¯•
    <BLANKLINE>
    >>> seg.default_config('ar') # doctest: +SKIP
    >>> sent = u'Ù‡Ø°Ø§ Ù‡Ùˆ ØªØµÙ†ÙŠÙ� Ø³ØªØ§Ù†Ù�ÙˆØ±Ø¯ Ø§Ù„Ø¹Ø±Ø¨ÙŠ Ù„Ù„ÙƒÙ„Ù…Ø§Øª'
    >>> print(seg.segment(sent.split())) # doctest: +SKIP
    Ù‡Ø°Ø§ Ù‡Ùˆ ØªØµÙ†ÙŠÙ� Ø³ØªØ§Ù†Ù�ÙˆØ±Ø¯ Ø§Ù„Ø¹Ø±Ø¨ÙŠ Ù„ Ø§Ù„ÙƒÙ„Ù…Ø§Øª
    <BLANKLINE>
    zstanford-segmenter.jarNÚfalseúUTF-8Fú-mx2gc                 ó„  — t          j        dt          ¦  «         t          j        t	          d¦  «        t          d¬¦  «         t          j        dt          ¦  «         t          | j        |ddt          |¬¦  «        }|�t          d	|d
dt          |¬¦  «        }nd }t          j	         
                    d„ ||fD ¦   «         ¦  «        | _        || _        || _        || _        || _        || _        || _        |	| _        || _        |
€i n|
}
d 
                    d„ |
                     ¦   «         D ¦   «         ¦  «        | _        d S )NÚalwaysz}
The StanfordTokenizer will be deprecated in version 3.2.5.
Please use [91mnltk.parse.corenlp.CoreNLPTokenizer[0m instead.'é   )Ú
stacklevelÚignore©ÚSTANFORD_SEGMENTER© )Úenv_varsÚ
searchpathÚurlÚverbosezslf4j-api.jar)ÚSLF4Jr   c              3   ó   K  — | ]}|®|V — Œ	d S ©Nr   )Ú.0Ú_s     úT/var/www/piapp/venv/lib/python3.11/site-packages/nltk/tokenize/stanford_segmenter.pyú	<genexpr>z-StanfordSegmenter.__init__.<locals>.<genexpr>j   s+   è è € ð -
ð -
Ø°a°mˆA°m°m°m°mð-
ð -
ó    ú,c              3   óN   K  — | ] \  }}|› d t          j        |¦  «        › �V — Œ!dS )Ú=N)ÚjsonÚdumps)r   ÚkeyÚvals      r!   r"   z-StanfordSegmenter.__init__.<locals>.<genexpr>x   sM   è è € ð %
ð %
Ù+3¨3°ˆsÐ&Ð&•T”Z ‘_”_Ð&Ð&ð%
ð %
ð %
ð %
ð %
ð %
r#   )ÚwarningsÚsimplefilterÚDeprecationWarningÚwarnÚstrr   Ú_JARÚ_stanford_urlÚosÚpathsepÚjoinÚ_stanford_jarÚ_java_classÚ_modelÚ_sihan_corpora_dictÚ_sihan_post_processingÚ_keep_whitespacesÚ_dictÚ	_encodingÚjava_optionsÚitemsÚ_options_cmd)ÚselfÚpath_to_jarÚpath_to_slf4jÚ
java_classÚpath_to_modelÚpath_to_dictÚpath_to_sihan_corpora_dictÚsihan_post_processingÚkeep_whitespacesÚencodingÚoptionsr   r=   Ústanford_segmenterÚslf4js                  r!   Ú__init__zStanfordSegmenter.__init__8   s‰  € õ  	Ô˜hÕ(:Ñ;Ô;Ð;ÝŒÝðZñô õ
 Øð	
ñ 	
ô 	
ð 	
õ 	Ô˜hÕ(:Ñ;Ô;Ð;å%ØŒIØØ,ØÝØð
ñ 
ô 
Ðð Ð$ÝØØØ8ØÝ!Øðñ ô ˆEˆEð ˆEõ  œZŸ_š_ð -
ð -
Ø*¨EÐ2ð-
ñ -
ô -
ñ 
ô 
ˆÔð &ˆÔØ#ˆŒØ#=ˆÔ Ø&;ˆÔ#Ø!1ˆÔØ!ˆŒ
à!ˆŒØ(ˆÔØ˜�"�"¨WˆØŸHšHð %
ð %
Ø7>·}²}±´ð%
ñ %
ô %
ñ 
ô 
ˆÔÐÐr#   c                 ó@  — d}t           j                             d¦  «        r>t           j                             t           j                             d¦  «        d¦  «        h}d| _        d| _        d| _        |dk    r
d| _        d}nÕ|d	k    r½d
| _        d}d| _        d}	 t          ||t          dd¬¦  «        | _        n%# t          $ r}t          d|z  ¦  «        |‚d}~ww xY wd}	 t          |t          dd¬¦  «        }t           j                             ||¦  «        | _        n7# t          $ r}t          d|z  ¦  «        |‚d}~ww xY wt          d|› �¦  «        ‚	 t          ||t          dd¬¦  «        | _        dS # t          $ r}t          d|z  ¦  «        |‚d}~ww xY w)z©
        Attempt to initialize Stanford Word Segmenter for the specified language
        using the STANFORD_SEGMENTER and STANFORD_MODELS environment variables
        r   r   ÚdataNr   Úarz=edu.stanford.nlp.international.arabic.process.ArabicSegmenterz'arabic-segmenter-atb+bn+arztrain.ser.gzÚzhz%edu.stanford.nlp.ie.crf.CRFClassifierzpku.gzÚtruezdict-chris6.ser.gzF)ÚSTANFORD_MODELS)r   r   r   r   z_Could not find '%s' (tried using env. variables STANFORD_MODELS and <STANFORD_SEGMENTER>/data/)z./data/r   )r   r   r   zMCould not find '%s' (tried using the STANFORD_SEGMENTER environment variable)zUnsupported language )rS   r   )r2   ÚenvironÚgetÚpathr4   r;   r8   r9   r6   r   r1   ÚLookupErrorr   r7   )r@   ÚlangÚsearch_pathÚmodelrE   ÚeÚ	sihan_dirÚpath_to_sihan_dirs           r!   Údefault_configz StanfordSegmenter.default_config|   s0  € ð ˆÝŒ:�>Š>Ð.Ñ/Ô/ð 	WÝœ7Ÿ<š<­¬
¯ªÐ7KÑ(LÔ(LÈfÑUÔUÐVˆKð ˆŒ
Ø#'ˆÔ Ø&-ˆÔ#à�4Š<ˆ<àOð Ôð >ˆEˆEà�TŠ\ˆ\ØFˆDÔØˆEØ*0ˆDÔ'à/ˆLðÝ&Ø Ø*Ý%Ø!Ø1ðñ ô �”
�
øõ ð ð ð Ý!ðPà"ñ#ñô ð ð	øøøøðøøøð "ˆIðÝ$,ØÝ%Ø!Ø4ð	%ñ %ô %Ð!õ ,.¬7¯<ª<Ð8IÈ9Ñ+UÔ+U�Ô(Ð(øÝð ð ð Ý!ð?ØAJñKñô ð ðøøøøðøøøõ Ð<°dÐ<Ð<Ñ=Ô=Ð=ð	Ý#ØØ&Ý!ØØBðñ ô ˆDŒKˆKˆKøõ ð 	ð 	ð 	ÝðLØNSñTñô ð ðøøøøð	øøøsH   ÂB= Â=
CÃCÃCÃ%=D# Ä#
EÄ-E Å EÅE; Å;
FÆFÆFc                 óJ   •— t          ¦   «                              |¦  «         d S r   )ÚsuperÚtokenize)r@   ÚsÚ	__class__s     €r!   ra   zStanfordSegmenter.tokenizeÅ   s!   ø€ Ý‰Œ×Ò˜ÑÔÐÐÐr#   c                 óÂ   — | j         d| j        d| j        d|g}| j        �*|                     d| j        d| j        d| j        g¦  «         |                      |¦  «        }|S )Ú ú-loadClassifierú-keepAllWhitespacesú	-textFileNú-serDictionaryú-sighanCorporaDictú-sighanPostProcessing)r6   r7   r:   r8   Úextendr;   r9   Ú_execute)r@   Úinput_file_pathÚcmdÚstdouts       r!   Úsegment_filezStanfordSegmenter.segment_fileÈ   s€   € ð ÔØØŒKØ!ØÔ"ØØð
ˆð Ô#Ð/Ø�JŠJà$Ø”JØ(ØÔ,Ø+ØÔ/ðñ	ô 	ð 	ð —’˜sÑ#Ô#ˆàˆr#   c                 ó.   — |                       |g¦  «        S r   )Úsegment_sents)r@   Útokenss     r!   ÚsegmentzStanfordSegmenter.segmentã   s   € Ø×!Ò! 6 (Ñ+Ô+Ð+r#   c                 óX  — | j         }t          j        d¬¦  «        \  }| _        t	          j        |d¦  «        }d                     d„ |D ¦   «         ¦  «        }t          |t          ¦  «        r|r| 	                    |¦  «        }| 
                    |¦  «         |                     ¦   «          | j        d| j        d| j        d| j        g}| j        �*|                     d
| j        d| j        d| j        g¦  «         |                      |¦  «        }t	          j        | j        ¦  «         |S )re   T)ÚtextÚwbÚ
c              3   ó@   K  — | ]}d                       |¦  «        V — ŒdS )re   N)r4   )r   Úxs     r!   r"   z2StanfordSegmenter.segment_sents.<locals>.<genexpr>î   s,   è è € Ð:Ð:¨1˜3Ÿ8š8 A™;œ;Ð:Ð:Ð:Ð:Ð:Ð:r#   rf   rg   rh   Nri   rj   rk   )r<   ÚtempfileÚmkstempÚ_input_file_pathr2   Úfdopenr4   Ú
isinstancer/   ÚencodeÚwriteÚcloser6   r7   r:   r8   rl   r;   r9   rm   Úunlink)r@   Ú	sentencesrI   Ú	_input_fhÚ_inputro   rp   s          r!   rs   zStanfordSegmenter.segment_sentsæ   s9  € à”>ˆå+3Ô+;ÀÐ+FÑ+FÔ+FÑ(ˆ	�4Ô(õ ”I˜i¨Ñ.Ô.ˆ	Ø—’Ð:Ð:°	Ð:Ñ:Ô:Ñ:Ô:ˆÝ�f�cÑ"Ô"ð 	- xð 	-Ø—]’] 8Ñ,Ô,ˆFØ�Š˜ÑÔÐØ�ŠÑÔÐð ÔØØŒKØ!ØÔ"ØØÔ!ð
ˆð Ô#Ð/Ø�JŠJà$Ø”JØ(ØÔ,Ø+ØÔ/ðñ	ô 	ð 	ð —’˜sÑ#Ô#ˆõ 	Œ	�$Ô'Ñ(Ô(Ð(àˆr#   c                 ó‚  — | j         }|                     d|g¦  «         | j        }|r|                     d| j        g¦  «         d                     t          ¦  «        }t          | j        |¬¦  «         t          || j        t          t          ¬¦  «        \  }}| 
                    |¦  «        }t          |d¬¦  «         |S )Nz-inputEncodingz-optionsre   )rJ   r   )Ú	classpathrp   ÚstderrF)r<   rl   r?   r4   r   r   r=   r	   r5   r   Údecode)r@   ro   r   rI   r?   Údefault_optionsrp   Ú_stderrs           r!   rm   zStanfordSegmenter._execute  sÅ   € Ø”>ˆØ�
Š
Ð$ hÐ/Ñ0Ô0Ð0ØÔ(ˆØð 	8Ø�JŠJ˜
 DÔ$5Ð6Ñ7Ô7Ð7àŸ(š(¥=Ñ1Ô1ˆõ 	˜DÔ-°wÐ?Ñ?Ô?Ð?åØ˜4Ô-µdÅ4ð
ñ 
ô 
‰ˆ�ð —’˜xÑ(Ô(ˆõ 	˜O°UÐ;Ñ;Ô;Ð;àˆr#   )NNNNNNr   r   r   NFr   )F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r0   rM   r^   ra   rq   ru   rs   rm   Ú__classcell__)rc   s   @r!   r   r       sÝ   ø€ € € € € ðð ð* $€Dð ØØØØØ#'Ø%Ø ØØØØðB
ð B
ð B
ð B
ðHGð Gð GðRð ð ð ð ðð ð ð6,ð ,ð ,ð(ð (ð (ðTð ð ð ð ð ð ð r#   r   )r'   r2   r|   r+   Ú
subprocessr   Únltk.internalsr   r   r   r   r   r	   Únltk.tokenize.apir
   r1   r   r   r#   r!   ú<module>r–      sä   ðð €€€Ø 	€	€	€	Ø €€€Ø €€€Ø Ð Ð Ð Ð Ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð )Ð (Ð (Ð (Ð (Ð (à3€ðDð Dð Dð Dð D˜
ñ Dô Dð Dð Dð Dr#   