§
    'ê[fŽ:  ã                   ó:   — d Z ddlZddlmZ  G d„ de¦  «        ZdS )u^  
ISRI Arabic Stemmer

The algorithm for this stemmer is described in:

Taghva, K., Elkoury, R., and Coombs, J. 2005. Arabic Stemming without a root dictionary.
Information Science Research Institute. University of Nevada, Las Vegas, USA.

The Information Science Research Instituteâ€™s (ISRI) Arabic stemmer shares many features
with the Khoja stemmer. However, the main difference is that ISRI stemmer does not use root
dictionary. Also, if a root is not found, ISRI stemmer returned normalized form, rather than
returning the original unmodified word.

Additional adjustments were made to improve the algorithm:

1- Adding 60 stop words.
2- Adding the pattern (ØªÙ�Ø§Ø¹ÙŠÙ„) to ISRI pattern set.
3- The step 2 in the original algorithm was normalizing all hamza. This step is discarded because it
increases the word ambiguities and changes the original root.

é    N)ÚStemmerIc                   ón   — e Zd ZdZd„ Zd„ Zdd„Zd„ Zd„ Zd„ Z	d	„ Z
d
„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ ZdS )ÚISRIStemmera&  
    ISRI Arabic stemmer based on algorithm: Arabic Stemming without a root dictionary.
    Information Science Research Institute. University of Nevada, Las Vegas, USA.

    A few minor modifications have been made to ISRI basic algorithm.
    See the source code of this module for more information.

    isri.stem(token) returns Arabic root for the given token.

    The ISRI Stemmer requires that all tokens have Unicode string types.
    If you use Python IDLE on Arabic Windows you have to decode text first
    using Arabic '1256' coding.
    c                 óv  — g d¢| _         ddg| _        g d¢| _        g d¢| _        g d¢| _        g d¢| _        dgd	gg d
¢dgdœ| _        d	dgg d¢g d¢g d¢ddgd	dgd	dgdœ| _        t          j	        d¦  «        | _
        t          j	        d¦  «        | _        t          j	        d¦  «        | _        g d¢| _        d S )N)u   ÙƒØ§Ù„u   Ø¨Ø§Ù„u   ÙˆÙ„Ù„u   ÙˆØ§Ù„u   Ø§Ù„u   Ù„Ù„)	u   Ù„u   Ø¨u   Ù�u   Ø³õ   Ùˆõ   ÙŠõ   Øªõ   Ù†õ   Ø§)u   ØªÙ…Ù„u   Ù‡Ù…Ù„u   ØªØ§Ù†u   ØªÙŠÙ†u   ÙƒÙ…Ù„)u   ÙˆÙ†u   Ø§Øªu   Ø§Ù†u   ÙŠÙ†u   ØªÙ†u   ÙƒÙ…u   Ù‡Ù†u   Ù†Ø§u   ÙŠØ§u   Ù‡Ø§u   ØªÙ…u   ÙƒÙ†u   Ù†ÙŠu   ÙˆØ§u   Ù…Ø§u   Ù‡Ù…)õ   Ø©u   Ù‡r   u   Ùƒr	   r   r
   õ   Ù…r   )r   r   r   r   )r   é   é   é   r	   )r   r   r   )r   r	   r   )r   r   r	   r   )r   r   r   r   é   é   é   z[\u064B-\u0652]z[\u0621\u0624\u0626]z^[\u0622\u0623\u0625])1u   ÙŠÙƒÙˆÙ†u   ÙˆÙ„ÙŠØ³u   ÙˆÙƒØ§Ù†u   ÙƒØ°Ù„Ùƒu   Ø§Ù„ØªÙŠu   ÙˆØ¨ÙŠÙ†u
   Ø¹Ù„ÙŠÙ‡Ø§u   Ù…Ø³Ø§Ø¡u   Ø§Ù„Ø°ÙŠu
   ÙˆÙƒØ§Ù†Øªu   ÙˆÙ„ÙƒÙ†u
   ÙˆØ§Ù„ØªÙŠu   ØªÙƒÙˆÙ†u
   Ø§Ù„ÙŠÙˆÙ…u   Ø§Ù„Ù„Ø°ÙŠÙ†u   Ø¹Ù„ÙŠÙ‡u   ÙƒØ§Ù†Øªu   Ù„Ø°Ù„Ùƒu   Ø£Ù…Ø§Ù…u   Ù‡Ù†Ø§Ùƒu   Ù…Ù†Ù‡Ø§u
   Ù…Ø§Ø²Ø§Ù„u
   Ù„Ø§Ø²Ø§Ù„u   Ù„Ø§ÙŠØ²Ø§Ù„u   Ù…Ø§ÙŠØ²Ø§Ù„u   Ø§ØµØ¨Ø­u   Ø£ØµØ¨Ø­u   Ø£Ù…Ø³Ù‰u   Ø§Ù…Ø³Ù‰u   Ø£Ø¶Ø­Ù‰u   Ø§Ø¶Ø­Ù‰u
   Ù…Ø§Ø¨Ø±Ø­u
   Ù…Ø§Ù�ØªØ¦u   Ù…Ø§Ø§Ù†Ù�Ùƒu   Ù„Ø§Ø³ÙŠÙ…Ø§u   ÙˆÙ„Ø§ÙŠØ²Ø§Ù„u   Ø§Ù„Ø­Ø§Ù„ÙŠu
   Ø§Ù„ÙŠÙ‡Ø§u
   Ø§Ù„Ø°ÙŠÙ†u   Ù�Ø§Ù†Ù‡u
   ÙˆØ§Ù„Ø°ÙŠu   ÙˆÙ‡Ø°Ø§u   Ù„Ù‡Ø°Ø§u   Ù�ÙƒØ§Ù†u
   Ø³ØªÙƒÙˆÙ†u   Ø§Ù„ÙŠÙ‡u   ÙŠÙ…ÙƒÙ†u   Ø¨Ù‡Ø°Ø§u   Ø§Ù„Ø°Ù‰)Úp3Úp2Úp1Ús3Ús2Ús1Úpr4Úpr53ÚreÚcompileÚre_short_vowelsÚre_hamzaÚre_initial_hamzaÚ
stop_words)Úselfs    úB/var/www/piapp/venv/lib/python3.11/site-packages/nltk/stem/isri.pyÚ__init__zISRIStemmer.__init__3   s0  € ð
ð 
ð 
ˆŒð " >Ð2ˆŒð

ð 

ð 

ˆŒð
ð 
ð 
ˆŒð
ð 
ð 
ˆŒð( YÐXÐXˆŒð ˆzØˆzØ-Ð-Ð-Øˆzð	
ð 
ˆŒð ˜(Ð#Ø-Ð-Ð-Ø-Ð-Ð-Ø-Ð-Ð-Ø˜(Ð#Ø˜(Ð#Ø˜(Ð#ð
ð 
ˆŒ	õ  "œzÐ*<Ñ=Ô=ˆÔÝœ
Ð#:Ñ;Ô;ˆŒÝ "¤
Ð+CÑ DÔ DˆÔð2
ð 2
ð 2
ˆŒˆˆó    c                 óX  — |                       |d¦  «        }|| j        v r|S |                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                       |d¦  «        }t          |¦  «        dk    r|                      |¦  «        }�n	t          |¦  «        dk    r+|                      |¦  «        }|                      |¦  «        }nËt          |¦  «        dk    r+|  	                    |¦  «        }|  
                    |¦  «        }n�t          |¦  «        dk    rz|                      |¦  «        }t          |¦  «        dk    r|                      |¦  «        }t          |¦  «        dk    r*|  	                    |¦  «        }|  
                    |¦  «        }|S )z?
        Stemming a word token using the ISRI stemmer.
        r   r   r   r   r   é   )Únormr!   Úpre32Úsuf32ÚwawÚlenÚpro_w4Úpro_w53Úend_w5Úpro_w6Úend_w6Úsuf1Úpre1)r"   Útokens     r#   ÚstemzISRIStemmer.stem·   s‰  € ð —	’	Ø�1ñ
ô 
ˆð �D”OÐ#Ð#ØˆLØ—
’
Øñ
ô 
ˆð —
’
Øñ
ô 
ˆð —’Øñ
ô 
ˆð —	’	˜% Ñ#Ô#ˆåˆu‰:Œ:˜Š?ˆ?Ø—K’K Ñ&Ô&ˆE‰EÝ�‰ZŒZ˜1Š_ˆ_Ø—L’L Ñ'Ô'ˆEØ—K’K Ñ&Ô&ˆEˆEÝ�‰ZŒZ˜1Š_ˆ_Ø—K’K Ñ&Ô&ˆEØ—K’K Ñ&Ô&ˆEˆEÝ�‰ZŒZ˜1Š_ˆ_Ø—I’I˜eÑ$Ô$ˆEÝ�5‰zŒz˜QŠˆØŸ	š	 %Ñ(Ô(�Ý�5‰zŒz˜QŠˆØŸš EÑ*Ô*�ØŸš EÑ*Ô*�Øˆr%   r   c                 ó  — |dk    r| j                              d|¦  «        }n^|dk    r| j                             d|¦  «        }n<|dk    r6| j                              d|¦  «        }| j                             d|¦  «        }|S )zƒ
        normalization:
        num=1  normalize diacritics
        num=2  normalize initial hamza
        num=3  both 1&2
        r   Ú r   r   r   )r   Úsubr    )r"   ÚwordÚnums      r#   r(   zISRIStemmer.normÜ   sˆ   € ð �!Š8ˆ8ØÔ'×+Ò+¨B°Ñ5Ô5ˆDˆDØ�AŠXˆXØÔ(×,Ò,¨X°tÑ<Ô<ˆDˆDØ�AŠXˆXØÔ'×+Ò+¨B°Ñ5Ô5ˆDØÔ(×,Ò,¨X°tÑ<Ô<ˆDØˆr%   c                 óþ   — t          |¦  «        dk    r+| j        D ]#}|                     |¦  «        r|dd…         c S Œ$t          |¦  «        dk    r+| j        D ]#}|                     |¦  «        r|dd…         c S Œ$|S )z9remove length three and length two prefixes in this orderr   r   Nr   r   )r,   r   Ú
startswithr   )r"   r9   Úpre3Úpre2s       r#   r)   zISRIStemmer.pre32ì   s›   € åˆt‰9Œ9˜Š>ˆ>Øœð $ð $�Ø—?’? 4Ñ(Ô(ð $Ø   œ8�O�O�Oð$åˆt‰9Œ9˜Š>ˆ>Øœð $ð $�Ø—?’? 4Ñ(Ô(ð $Ø   œ8�O�O�Oð$àˆr%   c                 óþ   — t          |¦  «        dk    r+| j        D ]#}|                     |¦  «        r|dd…         c S Œ$t          |¦  «        dk    r+| j        D ]#}|                     |¦  «        r|dd…         c S Œ$|S )z9remove length three and length two suffixes in this orderr   Néýÿÿÿr   éþÿÿÿ)r,   r   Úendswithr   )r"   r9   Úsuf3Úsuf2s       r#   r*   zISRIStemmer.suf32ø   s¡   € åˆt‰9Œ9˜Š>ˆ>Øœð %ð %�Ø—=’= Ñ&Ô&ð %Ø   œ9Ð$Ð$Ð$ð%åˆt‰9Œ9˜Š>ˆ>Øœð %ð %�Ø—=’= Ñ&Ô&ð %Ø   œ9Ð$Ð$Ð$ð%àˆr%   c                 ó\   — t          |¦  «        dk    r|dd…         dk    r
|dd…         }|S )uH   remove connective â€˜Ùˆâ€™ if it precedes a word beginning with â€˜Ùˆâ€™r   Nr   u   ÙˆÙˆr   )r,   ©r"   r9   s     r#   r+   zISRIStemmer.waw  s5   € åˆt‰9Œ9˜Š>ˆ>˜d 2 A 2œh¨.Ò8Ð8Ø˜˜˜”8ˆDØˆr%   c                 ó¨  — |d         | j         d         v r|dd…         }n±|d         | j         d         v r|dd…         |dd…         z   }n†|d         | j         d         v r|dd…         |d         z   }n]|d         | j         d         v r|dd…         }n=|                      |¦  «        }t          |¦  «        dk    r|                      |¦  «        }|S )z;process length four patterns and extract length three rootsr   r   Nr   r   éÿÿÿÿr   )r   r2   r,   r3   rF   s     r#   r-   zISRIStemmer.pro_w4
  sÚ   € à�Œ7�d”h˜q”kÐ!Ð!Ø˜˜˜”8ˆDˆDØ�!ŒW˜œ œÐ#Ð#Ø˜˜˜”8˜d 1 2 2œhÑ&ˆDˆDØ�!ŒW˜œ œÐ#Ð#Ø˜˜˜”8˜d 1œgÑ%ˆDˆDØ�!ŒW˜œ œÐ#Ð#Ø˜˜˜”9ˆDˆDà—9’9˜T‘?”?ˆDÝ�4‰yŒy˜AŠ~ˆ~Ø—y’y ‘”�Øˆr%   c                 ó  — |d         | j         d         v r!|d         dk    r|d         |dd…         z   }�nÐ|d         | j         d         v r!|d         dk    r|dd…         |d         z   }�nš|d         | j         d         v r|d         d	k    r|dd…         }�nm|d         | j         d         v r!|d         d
k    r|d         |dd…         z   }�n7|d         | j         d         v r!|d         dk    r|d         |dd…         z   }�n|d         | j         d         v r!|d         d	k    r|dd…         |d         z   }�nË|d         | j         d         v r|d         dk    r|dd…         }�nž|d         dk    r!|d         dk    r|dd…         |d         z   }�nq|d         dk    r|d         dk    r|dd…         }�nM|d         dk    r!|d         d
k    r|dd…         |d         z   }�n |d         dk    r'|d         dk    r|d         |d         z   |d         z   }ní|d         dk    r |d         dk    r|d         |dd…         z   }nÁ|d         dk    r |d         dk    r|dd…         |d         z   }n•|d         d	k    r |d         dk    r|d         |dd…         z   }ni|d         dk    r |d         dk    r|dd…         |d         z   }n=|                      |¦  «        }t          |¦  «        dk    r|                      |¦  «        }|S )z;process length five patterns and extract length three rootsr   r   r   r   r   Nr   r   r   r	   r   r   r
   r   r   u   Ø¦)r   r2   r,   r3   rF   s     r#   r.   zISRIStemmer.pro_w53  sc  € à�Œ7�d”i ”lÐ"Ð" t¨A¤w°(Ò':Ð':Ø˜”7˜T ! " "œXÑ%ˆD‰DØ�!ŒW˜œ	 !œÐ$Ð$¨¨a¬°HÒ)<Ð)<Ø˜˜!˜”9˜t AœwÑ&ˆD‰DØ�!ŒW˜œ	 !œÐ$Ð$¨¨a¬°HÒ)<Ð)<Ø˜˜!˜”9ˆD‰DØ�!ŒW˜œ	 !œÐ$Ð$¨¨a¬°HÒ)<Ð)<Ø˜”7˜T ! " "œXÑ%ˆD‰DØ�!ŒW˜œ	 !œÐ$Ð$¨¨a¬°HÒ)<Ð)<Ø˜”7˜T ! " "œXÑ%ˆD‰DØ�!ŒW˜œ	 !œÐ$Ð$¨¨a¬°HÒ)<Ð)<Ø˜˜˜”8˜d 1œgÑ%ˆD‰DØ�!ŒW˜œ	 !œÐ$Ð$¨¨a¬°HÒ)<Ð)<Ø˜˜˜”8ˆD‰DØ�!ŒW˜Ò Ð  T¨!¤W°Ò%8Ð%8Ø˜˜!˜”9˜t AœwÑ&ˆD‰DØ�!ŒW˜Ò Ð  T¨!¤W°Ò%8Ð%8Ø˜˜˜”8ˆD‰DØ�!ŒW˜Ò Ð  T¨!¤W°Ò%8Ð%8Ø˜˜!˜”9˜t AœwÑ&ˆD‰DØ�!ŒW˜Ò Ð  T¨!¤W°Ò%8Ð%8Ø˜”7˜T !œWÑ$ t¨A¤wÑ.ˆDˆDØ�!ŒW˜Ò Ð  T¨!¤W°Ò%8Ð%8Ø˜”7˜T ! " "œXÑ%ˆDˆDØ�!ŒW˜Ò Ð  T¨!¤W°Ò%8Ð%8Ø˜˜˜”8˜d 1œgÑ%ˆDˆDØ�!ŒW˜Ò Ð  T¨!¤W°Ò%8Ð%8Ø˜”7˜T ! A #œYÑ&ˆDˆDØ�!ŒW˜Ò Ð  T¨!¤W°Ò%8Ð%8Ø˜˜˜”8˜d 1œgÑ%ˆDˆDà—9’9˜T‘?”?ˆDÝ�4‰yŒy˜AŠ~ˆ~Ø—y’y ‘”�Øˆr%   c                 ó¶   — |d         | j         d         v r|dd…         }n8|d         dk    r|dd…         }n!|d         dk    r|dd…         |dd…         z   }|S )	z:process length five patterns and extract length four rootsr   r   r   Nr   r   r   r   )r   rF   s     r#   Úpro_w54zISRIStemmer.pro_w54@  sr   € à�Œ7�d”i ”lÐ"Ð"Ø˜˜˜”8ˆDˆDØ�!ŒW˜Ò Ð Ø˜˜˜”8ˆDˆDØ�!ŒW˜Ò Ð Ø˜˜˜”8˜d 1 2 2œhÑ&ˆDØˆr%   c                 ó¨   — t          |¦  «        dk    r|                      |¦  «        }n(t          |¦  «        dk    r|                      |¦  «        }|S )z!ending step (word of length five)r   r   )r,   r-   rK   rF   s     r#   r/   zISRIStemmer.end_w5J  sI   € åˆt‰9Œ9˜Š>ˆ>Ø—;’;˜tÑ$Ô$ˆDˆDÝ�‰YŒY˜!Š^ˆ^Ø—<’< Ñ%Ô%ˆDØˆr%   c                 óÔ  — |                      d¦  «        s|                      d¦  «        r|dd…         }�n1|d         dk    r,|d         dk    r |d         d	k    r|d
d…         |d         z   }nù|d         dk    r3|d         dk    r'|d         dk    r|d
         |d         z   |d         z   }nº|d         dk    r2|d         dk    r&|d         |d         k    r|d
         |dd…         z   }n||d         dk    r3|d         dk    r'|d         dk    r|d
         |d         z   |d         z   }n=|                      |¦  «        }t          |¦  «        dk    r|                      |¦  «        }|S )z:process length six patterns and extract length three rootsu   Ø§Ø³Øªu   Ù…Ø³Øªr   Nr   r   r   r   r   r   r   r   r	   r   r   r   )r<   r2   r,   r3   rF   s     r#   r0   zISRIStemmer.pro_w6R  sŒ  € à�?Š?Ð/Ñ0Ô0ð 	'°D·O²OØ ñ5
ô 5
ð 	'ð ˜˜˜”8ˆD‰Dà�ŒG�xÒÐ D¨¤G¨xÒ$7Ð$7¸DÀ¼GÀxÒ<OÐ<Oà˜˜!˜”9˜t AœwÑ&ˆDˆDà�ŒG�xÒÐ D¨¤G¨xÒ$7Ð$7¸DÀ¼GÀxÒ<OÐ<Oà˜”7˜T !œWÑ$ t¨A¤wÑ.ˆDˆDà�ŒG�xÒÐ D¨¤G¨xÒ$7Ð$7¸DÀ¼GÀtÈAÄwÒ<NÐ<Nà˜”7˜T ! " "œXÑ%ˆDˆDà�ŒG�xÒÐ D¨¤G¨xÒ$7Ð$7¸DÀ¼GÀxÒ<OÐ<Oà˜”7˜T !œWÑ$ t¨A¤wÑ.ˆDˆDà—9’9˜T‘?”?ˆDÝ�4‰yŒy˜AŠ~ˆ~Ø—y’y ‘”�Øˆr%   c                 óœ   — |d         dk    r |d         dk    r|dd…         |d         z   }n|                      d¦  «        r
|dd…         }|S )	z9process length six patterns and extract length four rootsr   r   r   r   r   u   Ù…Øªr   N)r<   rF   s     r#   Úpro_w64zISRIStemmer.pro_w64n  s]   € à�Œ7�hÒÐ 4¨¤7¨hÒ#6Ð#6Ø˜˜!˜”9˜t AœwÑ&ˆDˆDØ�_Š_˜^Ñ,Ô,ð 	Ø˜˜˜”8ˆDØˆr%   c                 óÒ   — t          |¦  «        dk    r+|                      |¦  «        }|                      |¦  «        }n(t          |¦  «        dk    r|                      |¦  «        }|S )z ending step (word of length six)r   r   )r,   r.   r/   rO   rF   s     r#   r1   zISRIStemmer.end_w6v  sZ   € åˆt‰9Œ9˜Š>ˆ>Ø—<’< Ñ%Ô%ˆDØ—;’;˜tÑ$Ô$ˆDˆDÝ�‰YŒY˜!Š^ˆ^Ø—<’< Ñ%Ô%ˆDØˆr%   c                 ó\   — | j         D ]#}|                     |¦  «        r|dd…         c S Œ$|S )znormalize short sufixNrH   )r   rB   )r"   r9   Úsf1s      r#   r2   zISRIStemmer.suf1  sE   € à”7ð 	!ð 	!ˆCØ�}Š}˜SÑ!Ô!ð !Ø˜C˜R˜C”yÐ Ð Ð ð!àˆr%   c                 ó\   — | j         D ]#}|                     |¦  «        r|dd…         c S Œ$|S )znormalize short prefixr   N)r   r<   )r"   r9   Úsp1s      r#   r3   zISRIStemmer.pre1†  sB   € à”7ð 	 ð 	 ˆCØ�Š˜sÑ#Ô#ð  Ø˜A˜B˜B”x���ð àˆr%   N)r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r$   r5   r(   r)   r*   r+   r-   r.   rK   r/   r0   rO   r1   r2   r3   © r%   r#   r   r   $   s
  € € € € € ðð ðB
ð B
ð B
ðH#ð #ð #ðJð ð ð ð 
ð 
ð 
ð
ð 
ð 
ðð ð ðð ð ð $ð $ð $ðLð ð ðð ð ðð ð ð8ð ð ðð ð ðð ð ðð ð ð ð r%   r   )rX   r   Únltk.stem.apir   r   rY   r%   r#   ú<module>r[      sj   ððð ð* 
€	€	€	à "Ð "Ð "Ð "Ð "Ð "ðgð gð gð gð g�(ñ gô gð gð gð gr%   