§
    	 @g*   ã                   óê   — d Z ddlZddlmZ ddlZddlZ G d„ d¦  «        Z	 ddeee	f         ded	ee
         d
ee	         fd„Z	 dde
de	de
d	ee
         d
eee	f         f
d„Zdee         d
dfd„Zd„ ZdS )zJThis is an educational implementation of the byte pair encoding algorithm.é    N)ÚOptionalc                   óú   — e Zd Zdedeeef         ddfd„Zddedee         de	e         fd	„Z
d
e	e         defd„Zd
e	e         defd„Zd
e	e         de	e         fd„Zedededefd„¦   «         Zed„ ¦   «         ZdS )ÚSimpleBytePairEncodingÚpat_strÚmergeable_ranksÚreturnNc                óš   — || _         || _        d„ |                     ¦   «         D ¦   «         | _        t	          j        |¦  «        | _        dS )zCreates an Encoding object.c                 ó   — i | ]\  }}||“Œ	S © r   )Ú.0Útoken_bytesÚtokens      úI/var/www/piapp/venv/lib/python3.11/site-packages/tiktoken/_educational.pyú
<dictcomp>z3SimpleBytePairEncoding.__init__.<locals>.<dictcomp>   s   € Ð^Ð^Ð^Ñ0B°¸U˜ Ð^Ð^Ð^ó    N)r   r   ÚitemsÚ_decoderÚregexÚcompileÚ_pat)Úselfr   r   s      r   Ú__init__zSimpleBytePairEncoding.__init__   sI   € ð ˆŒà.ˆÔà^Ð^Ào×F[ÒF[ÑF]ÔF]Ð^Ñ^Ô^ˆŒÝ”M 'Ñ*Ô*ˆŒ	ˆ	ˆ	r   ÚcolourÚtextÚ	visualisec                 óÊ   — | j                              |¦  «        }g }|D ]C}|                     d¦  «        }t          | j        ||¬¦  «        }|                     |¦  «         ŒD|S )z`Encodes a string into tokens.

        >>> enc.encode("hello world")
        [388, 372]
        úutf-8)r   )r   ÚfindallÚencodeÚ
bpe_encoder   Úextend)r   r   r   ÚwordsÚtokensÚwordÚ
word_bytesÚword_tokenss           r   r   zSimpleBytePairEncoding.encode   sp   € ð ”	×!Ò! $Ñ'Ô'ˆØˆØð 	'ð 	'ˆDàŸš WÑ-Ô-ˆJÝ$ TÔ%9¸:ÐQZÐ[Ñ[Ô[ˆKØ�MŠM˜+Ñ&Ô&Ð&Ð&Øˆr   r#   c                 óF   ‡ — d                      ˆ fd„|D ¦   «         ¦  «        S )znDecodes a list of tokens into bytes.

        >>> enc.decode_bytes([388, 372])
        b'hello world'
        r   c              3   ó2   •K  — | ]}‰j         |         V — Œd S ©N©r   ©r   r   r   s     €r   ú	<genexpr>z6SimpleBytePairEncoding.decode_bytes.<locals>.<genexpr>+   s*   øè è € ÐAÐA°˜œ eÔ,ÐAÐAÐAÐAÐAÐAr   )Újoin©r   r#   s   ` r   Údecode_bytesz#SimpleBytePairEncoding.decode_bytes%   s,   ø€ ð �xŠxÐAÐAÐAÐA¸&ÐAÑAÔAÑAÔAÐAr   c                 óV   — |                       |¦  «                             dd¬¦  «        S )u   Decodes a list of tokens into a string.

        Decoded bytes are not guaranteed to be valid UTF-8. In that case, we replace
        the invalid bytes with the replacement character "ï¿½".

        >>> enc.decode([388, 372])
        'hello world'
        r   Úreplace©Úerrors)r/   Údecoder.   s     r   r4   zSimpleBytePairEncoding.decode-   s+   € ð × Ò  Ñ(Ô(×/Ò/°À	Ð/ÑJÔJÐJr   c                 ó    ‡ — ˆ fd„|D ¦   «         S )zÁDecodes a list of tokens into a list of bytes.

        Useful for visualising how a string is tokenised.

        >>> enc.decode_tokens_bytes([388, 372])
        [b'hello', b' world']
        c                 ó*   •— g | ]}‰j         |         ‘ŒS r   r*   r+   s     €r   ú
<listcomp>z>SimpleBytePairEncoding.decode_tokens_bytes.<locals>.<listcomp>@   s    ø€ Ð9Ð9Ð9¨�”˜eÔ$Ð9Ð9Ð9r   r   r.   s   ` r   Údecode_tokens_bytesz*SimpleBytePairEncoding.decode_tokens_bytes8   s   ø€ ð :Ð9Ð9Ð9°&Ð9Ñ9Ô9Ð9r   Útraining_dataÚ
vocab_sizec                 óH   — t          | ||¬¦  «        }t          ||¬¦  «        S )z#Train a BPE tokeniser on some data!)Údatar:   r   ©r   r   )Ú	bpe_trainr   )r9   r:   r   r   s       r   ÚtrainzSimpleBytePairEncoding.trainB   s,   € õ $¨À:ÐW^Ð_Ñ_Ô_ˆÝ%¨gÀÐWÑWÔWÐWr   c                 óŠ   — t          | t          ¦  «        rt          j        | ¦  «        } t	          | j        | j        ¬¦  «        S )Nr=   )Ú
isinstanceÚstrÚtiktokenÚget_encodingr   Ú_pat_strÚ_mergeable_ranks)Úencodings    r   Úfrom_tiktokenz$SimpleBytePairEncoding.from_tiktokenH   sF   € å�h¥Ñ$Ô$ð 	7ÝÔ,¨XÑ6Ô6ˆHÝ%ØÔ%°xÔ7Pð
ñ 
ô 
ð 	
r   ©r   )Ú__name__Ú
__module__Ú__qualname__rB   ÚdictÚbytesÚintr   r   Úlistr   r/   r4   r8   Ústaticmethodr?   rH   r   r   r   r   r   
   sU  € € € € € ð+ 3ð +¸¸eÀS¸jÔ9Ið +Èdð +ð +ð +ð +ðð ˜3ð ¨8°C¬=ð ÈÈSÌ	ð ð ð ð ð B 4¨¤9ð B°ð Bð Bð Bð Bð	K˜T #œYð 	K¨3ð 	Kð 	Kð 	Kð 	Kð:¨$¨s¬)ð :¸¸U¼ð :ð :ð :ð :ð ðX˜Sð X¨cð X¸Cð Xð Xð Xñ „\ðXð
 ð
ð 
ñ „\ð
ð 
ð 
r   r   r   r   Úinputr   r   c                 óò  ‡ — d„ |D ¦   «         }	 |r)|dv rt          |¦  «         n|dk    rt          |¦  «         d }d }t          t          |d d…         |dd …         ¦  «        ¦  «        D ]7\  }}‰                      |d         |d         z   ¦  «        }|�|�||k     r|}|}Œ8|€n3|€J ‚|d |…         ||         ||dz            z   gz   ||dz   d …         z   }ŒÊ|rt          ¦   «          ˆ fd	„|D ¦   «         }	|	S )
Nc                 ó.   — g | ]}t          |g¦  «        ‘ŒS r   ©rN   ©r   Úbs     r   r7   zbpe_encode.<locals>.<listcomp>T   s    € Ð'Ð'Ð'˜A�U�A�3‰ZŒZÐ'Ð'Ð'r   T©r   ÚcolorÚsimpleéÿÿÿÿé   r   é   c                 ó    •— g | ]
}‰|         ‘ŒS r   r   )r   Úpartr   s     €r   r7   zbpe_encode.<locals>.<listcomp>q   s   ø€ Ð6Ð6Ð6¨ˆo˜dÔ#Ð6Ð6Ð6r   )Úvisualise_tokensÚprintÚ	enumerateÚzipÚget)
r   rR   r   ÚpartsÚmin_idxÚmin_rankÚiÚpairÚrankr#   s
   `         r   r    r    Q   sf  ø€ ð (Ð' Ð'Ñ'Ô'€Eð_àð 	ØÐ/Ð/Ð/Ý  Ñ'Ô'Ð'Ð'Ø˜hÒ&Ð&Ý�e‘”�ð ˆØˆÝ ¥ U¨3¨B¨3¤Z°°q°r°r´Ñ!;Ô!;Ñ<Ô<ð 	 ð 	 ‰GˆAˆtØ"×&Ò& t¨A¤w°°a´Ñ'8Ñ9Ô9ˆDØÐ XÐ%5¸Àº¸Ø�Ø�øð ÐØØÐ"Ð"Ð"ð �h�w�h” 5¨¤>°E¸'ÀA¹+Ô4FÑ#FÐ"GÑGÈ%ÐPWÐZ[ÑP[ÐP]ÐP]ÔJ^Ñ^ˆð/_ð2 ð Ý‰Œˆà6Ð6Ð6Ð6°Ð6Ñ6Ô6€FØ€Mr   r<   r:   r   c                 ó.  ‡— |dk     rt          d¦  «        ‚i }t          d¦  «        D ]}||t          |g¦  «        <   Œd„ t          j        || ¦  «        D ¦   «         }t          |¦  «        |k     �r$t          j        ¦   «         Š|D ]5}t          |d d…         |dd …         ¦  «        D ]}‰|xx         dz  cc<   ŒŒ6t          ‰ˆfd„¬¦  «        }	|	d         |	d         z   }
t          |¦  «        }|||
<   g }|D ]Ê}g }d}|t          |¦  «        dz
  k     rh||         ||dz            f|	k    r| 
                    |
¦  «         |d	z  }n | 
                    ||         ¦  «         |dz  }|t          |¦  «        dz
  k     °h|t          |¦  «        dz
  k    r| 
                    ||         ¦  «         | 
                    |¦  «         ŒË|}|r¹t          d
|	d         › d|	d         › �¦  «         t          d|
› dt          |¦  «        › d�¦  «         |dv r1t          d¦  «         t          d„ |d d…         D ¦   «         ¦  «         n1|dk    r+t          d¦  «         |d d…         D ]}t          |¦  «         Œt          d¦  «         t          |¦  «        |k     �°$|S )Né   z;vocab_size must be at least 256, so we can encode all bytesc                 óL   — g | ]!}d „ |                      d¦  «        D ¦   «         ‘Œ"S )c                 ó.   — g | ]}t          |g¦  «        ‘ŒS r   rU   rV   s     r   r7   z(bpe_train.<locals>.<listcomp>.<listcomp>†   s    € Ð2Ð2Ð2˜��ˆs‰ŒÐ2Ð2Ð2r   r   )r   )r   r$   s     r   r7   zbpe_train.<locals>.<listcomp>…   s@   € ð  ð  ð  Ø7;Ð2Ð2˜TŸ[š[¨Ñ1Ô1Ð2Ñ2Ô2ð ð  ð  r   r[   r\   c                 ó   •— ‰|          S r)   r   )ÚxÚstatss    €r   ú<lambda>zbpe_train.<locals>.<lambda>‘   s   ø€ °E¸!´H€ r   )Úkeyr   r]   z The current most common pair is z + zSo we made z our zth tokenrX   z9Now the first fifty words in our training data look like:c                 ó   — g | ]	}|D ]}|‘ŒŒ
S r   r   )r   r$   r   s      r   r7   zbpe_train.<locals>.<listcomp>°   s%   € Ð!QÐ!QÐ!Q¨DÈDÐ!QÐ!QÀ5 %Ð!QÐ!QÐ!QÐ!Qr   é2   rZ   z:Now the first twenty words in our training data look like:é   ú
)Ú
ValueErrorÚrangerN   r   r   ÚlenÚcollectionsÚCounterrc   ÚmaxÚappendra   r`   )r<   r:   r   r   Úranksrh   r"   Úpieceri   Úmost_common_pairr   r   Ú	new_wordsr$   Únew_wordrq   s                  @r   r>   r>   u   s,  ø€ ð �DÒÐÝÐVÑWÔWÐWØ€EÝ�4‰[Œ[ð ð ˆØˆ�e�Q�C‰jŒjÑÐð ð  Ý?D¼}ÈWÐVZÑ?[Ô?[ð ñ  ô  €Eõ
 ˆe‰*Œ*�zÒ
!Ñ
!åÔ#Ñ%Ô%ˆØð 	!ð 	!ˆEÝ˜E # 2 #œJ¨¨a¨b¨b¬	Ñ2Ô2ð !ð !�Ø�d��”˜qÑ ��‘�ð!õ ˜uÐ*<Ð*<Ð*<Ð*<Ð=Ñ=Ô=ÐØ& qÔ)Ð,<¸QÔ,?Ñ?ˆÝ�E‘
”
ˆà"ˆˆkÑð ˆ	Øð 	'ð 	'ˆDØˆHØˆAØ•c˜$‘i”i !‘mÒ#Ð#Ø˜”G˜T ! a¡%œ[Ð)Ð-=Ò=Ð=à—O’O KÑ0Ô0Ð0Ø˜‘F�A�Aà—O’O D¨¤GÑ,Ô,Ð,Ø˜‘F�Að •c˜$‘i”i !‘mÒ#Ð#ð •C˜‘I”I ‘MÒ!Ð!Ø—’  Q¤Ñ(Ô(Ð(Ø×Ò˜XÑ&Ô&Ð&Ð&Øˆð ð 
	ÝÐbÐ5EÀaÔ5HÐbÐbÐM]Ð^_ÔM`ÐbÐbÑcÔcÐcÝÐF ÐFÐFµ#°e±*´*ÐFÐFÐFÑGÔGÐGØÐ/Ð/Ð/ÝÐQÑRÔRÐRÝ Ð!QÐ!Q°E¸#¸2¸#´JÐ!QÑ!QÔ!QÑRÔRÐRÐRØ˜hÒ&Ð&ÝÐRÑSÔSÐSØ! # 2 #œJð  ð  �DÝ˜$‘K”K�K�KÝ�$‰KŒKˆKõW ˆe‰*Œ*�zÒ
!Ñ
!ðZ €Lr   Útoken_valuesc                 ó8  — d„ dD ¦   «         }d„ | D ¦   «         }d}d }|D ]k}||t          |¦  «        z           }||k    r#||dz   t          |¦  «        z           }||k    sJ ‚|}|t          |¦  «        z  }t          ||z   d¬¦  «         Œlt          d¦  «         d S )	Nc                 ó   — g | ]}d |› d�‘Œ	S )z[48;5;Úmr   )r   rh   s     r   r7   z$visualise_tokens.<locals>.<listcomp>»   s$   € ÐPÐPÐP¨!Ð% Ð%Ð%Ð%ÐPÐPÐPr   )é§   é³   é¹   éM   éP   éD   é†   c                 ó<   — g | ]}|                      d d¬¦  «        ‘ŒS )r   r1   r2   )r4   )r   rp   s     r   r7   z$visualise_tokens.<locals>.<listcomp>¿   s(   € ÐVÐVÐVÀA˜AŸHšH W°Y˜HÑ?Ô?ÐVÐVÐVr   r   r\   Ú )Úendz[0m)rz   ra   )r„   Ú
backgroundÚunicode_token_valuesÚrunning_lengthÚ
last_colorr   rY   s          r   r`   r`   º   sÔ   € ØPÐPÐ/OÐPÑPÔP€Jð WÐVÈÐVÑVÔVÐà€NØ€JØ%ð %ð %ˆØ˜>­C°
©O¬OÑ;Ô<ˆØ�JÒÐØ °Ñ 2µc¸*±o´oÑEÔFˆEØ˜JÒ&Ð&Ð&Ð&Øˆ
Ø�#˜e™*œ*Ñ$ˆÝˆe�e‰m Ð$Ñ$Ô$Ð$Ð$Ý	ˆ+ÑÔÐÐÐr   c                  ó´  — d} t          t          d¦  «        5 }|                     ¦   «         }d d d ¦  «         n# 1 swxY w Y   t                               |d| ¬¦  «        }t          d¦  «         |                     d¦  «        }|                     |¦  «        dk    sJ ‚|                     |¦  «        dk    sJ ‚| 	                    |¦  «        dd	gk    sJ ‚|S )
NzN's|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+ÚriX  )r:   r   zJThis is the sequence of merges performed in order to encode 'hello world':zhello worlds   hello worlds   hellos    world)
ÚopenÚ__file__Úreadr   r?   ra   r   r4   r/   r8   )Úgpt2_patternÚfr<   Úencr#   s        r   Útrain_simple_encodingrž   Î   s  € à]ð õ 
�h˜Ñ	Ô	ð  Ø�vŠv‰xŒxˆðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð õ !×
&Ò
& t¸À\Ð
&Ñ
RÔ
R€Cå	Ð
VÑWÔWÐWØ�ZŠZ˜Ñ&Ô&€FØ�:Š:�fÑÔ Ò.Ð.Ð.Ð.Ø×Ò˜FÑ#Ô# ~Ò5Ð5Ð5Ð5Ø×"Ò" 6Ñ*Ô*¨x¸Ð.CÒCÐCÐCÐCà€Js   ˜9¹=Á =rI   )Ú__doc__r{   Útypingr   r   rC   r   rM   rN   rO   rB   rP   r    r>   r`   rž   r   r   r   ú<module>r¡      s]  ðØ PÐ PØ Ð Ð Ð Ø Ð Ð Ð Ð Ð à €€€à €€€ðD
ð D
ð D
ð D
ð D
ñ D
ô D
ð D
ðP QYð!ð !Ø˜% ˜*Ô%ð!Ø.3ð!Ø@HÈÄð!à	ˆ#„Yð!ð !ð !ð !ðJ JRðBð BØ
ðBØðBØ),ðBØ9AÀ#¼ðBà	ˆ%�ˆ*ÔðBð Bð Bð BðJ 4¨¤;ð °4ð ð ð ð ð(ð ð ð ð r   