o
    Í Ži,Q  ã                   @  sV  d dl mZ d dlmZ d dlmZ ddlmZmZm	Z	 ddl
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd„ dƒZG d	d
„ d
eƒZG dd„ deƒZG dd„ deƒZ G dd„ deƒZ!G dd„ deƒZ"G dd„ deƒZ#G dd„ deƒZ$G dd„ deƒZ%G dd„ deƒZ&edd�d/d"d#„ƒZ'ed$d�	&d0d1d,d-„ƒZ(d.S )2é    )Úannotations)Ú	lru_cache)Ú	getLoggeré   )ÚCOMMON_SAFE_ASCII_CHARACTERSÚTRACEÚUNICODE_SECONDARY_RANGE_KEYWORD)Úis_accentuatedÚ	is_arabicÚis_arabic_isolated_formÚis_case_variableÚis_cjkÚis_emoticonÚ	is_hangulÚis_hiraganaÚis_katakanaÚis_latinÚis_punctuationÚis_separatorÚ	is_symbolÚis_thaiÚis_unprintableÚremove_accentÚunicode_rangeÚis_cjk_uncommonc                   @  s<   e Zd ZdZddd„Zdd	d
„Zddd„Zeddd„ƒZdS )ÚMessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    Ú	characterÚstrÚreturnÚboolc                 C  ó   t ‚)z@
        Determine if given character should be fed in.
        N©ÚNotImplementedError©Úselfr   © r%   ú_C:\Users\psant\Downloads\Anu Raj Backend\school\venv\Lib\site-packages\charset_normalizer/md.pyÚeligible'   ó   zMessDetectorPlugin.eligibleÚNonec                 C  r    )z‰
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        Nr!   r#   r%   r%   r&   Úfeed-   s   zMessDetectorPlugin.feedc                 C  r    )zB
        Permit to reset the plugin to the initial state.
        Nr!   ©r$   r%   r%   r&   Úreset4   r(   zMessDetectorPlugin.resetÚfloatc                 C  r    )z…
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        Nr!   r+   r%   r%   r&   Úratio:   s   zMessDetectorPlugin.ratioN©r   r   r   r   ©r   r   r   r)   ©r   r)   ©r   r-   )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r'   r*   r,   Úpropertyr.   r%   r%   r%   r&   r   !   s    


r   c                   @  óB   e Zd Zddd„Zddd	„Zdd
d„Zddd„Zeddd„ƒZdS )Ú TooManySymbolOrPunctuationPluginr   r)   c                 C  s"   d| _ d| _d| _d | _d| _d S )Nr   F)Ú_punctuation_countÚ_symbol_countÚ_character_countÚ_last_printable_charZ_frenzy_symbol_in_wordr+   r%   r%   r&   Ú__init__D   s
   
z)TooManySymbolOrPunctuationPlugin.__init__r   r   r   c                 C  ó   |  ¡ S ©N©Úisprintabler#   r%   r%   r&   r'   L   ó   z)TooManySymbolOrPunctuationPlugin.eligiblec                 C  sp   |  j d7  _ || jkr3|tvr3t|ƒr|  jd7  _n| ¡ du r3t|ƒr3t|ƒdu r3|  jd7  _|| _d S )Nr   Fé   )	r<   r=   r   r   r:   Úisdigitr   r   r;   r#   r%   r%   r&   r*   O   s   ÿþ
ÿþ
ý
z%TooManySymbolOrPunctuationPlugin.feedc                 C  s   d| _ d| _d| _d S ©Nr   )r:   r<   r;   r+   r%   r%   r&   r,   a   ó   
z&TooManySymbolOrPunctuationPlugin.resetr-   c                 C  s0   | j dkrdS | j| j | j  }|dkr|S dS )Nr   ç        ç333333Ó?)r<   r:   r;   )r$   Zratio_of_punctuationr%   r%   r&   r.   f   s   

þz&TooManySymbolOrPunctuationPlugin.ratioNr1   r/   r0   r2   ©	r3   r4   r5   r>   r'   r*   r,   r7   r.   r%   r%   r%   r&   r9   C   s    



r9   c                   @  r8   )ÚTooManyAccentuatedPluginr   r)   c                 C  ó   d| _ d| _d S rF   ©r<   Ú_accentuated_countr+   r%   r%   r&   r>   s   ó   
z!TooManyAccentuatedPlugin.__init__r   r   r   c                 C  r?   r@   )Úisalphar#   r%   r%   r&   r'   w   rC   z!TooManyAccentuatedPlugin.eligiblec                 C  ó,   |  j d7  _ t|ƒr|  jd7  _d S d S ©Nr   )r<   r	   rN   r#   r%   r%   r&   r*   z   ó   ÿzTooManyAccentuatedPlugin.feedc                 C  rL   rF   rM   r+   r%   r%   r&   r,   €   rO   zTooManyAccentuatedPlugin.resetr-   c                 C  s*   | j dk rdS | j| j  }|dkr|S dS )Né   rH   gffffffÖ?rM   )r$   Zratio_of_accentuationr%   r%   r&   r.   „   s   
zTooManyAccentuatedPlugin.ratioNr1   r/   r0   r2   rJ   r%   r%   r%   r&   rK   r   s    



rK   c                   @  r8   )ÚUnprintablePluginr   r)   c                 C  rL   rF   )Ú_unprintable_countr<   r+   r%   r%   r&   r>   Ž   rO   zUnprintablePlugin.__init__r   r   r   c                 C  ó   dS ©NTr%   r#   r%   r%   r&   r'   ’   ó   zUnprintablePlugin.eligiblec                 C  s(   t |ƒr|  jd7  _|  jd7  _d S rR   )r   rV   r<   r#   r%   r%   r&   r*   •   s   zUnprintablePlugin.feedc                 C  s
   d| _ d S rF   )rV   r+   r%   r%   r&   r,   š   s   
zUnprintablePlugin.resetr-   c                 C  ó   | j dkrdS | jd | j  S )Nr   rH   rT   )r<   rV   r+   r%   r%   r&   r.   �   ó   
zUnprintablePlugin.ratioNr1   r/   r0   r2   rJ   r%   r%   r%   r&   rU   �   s    



rU   c                   @  r8   )ÚSuspiciousDuplicateAccentPluginr   r)   c                 C  ó   d| _ d| _d | _d S rF   ©Ú_successive_countr<   Ú_last_latin_characterr+   r%   r%   r&   r>   ¦   s   
z(SuspiciousDuplicateAccentPlugin.__init__r   r   r   c                 C  s   |  ¡ ot|ƒS r@   )rP   r   r#   r%   r%   r&   r'   ¬   s   z(SuspiciousDuplicateAccentPlugin.eligiblec                 C  st   |  j d7  _ | jd ur5t|ƒr5t| jƒr5| ¡ r%| j ¡ r%|  jd7  _t|ƒt| jƒkr5|  jd7  _|| _d S rR   )r<   r`   r	   Úisupperr_   r   r#   r%   r%   r&   r*   ¯   s   ÿþý
z$SuspiciousDuplicateAccentPlugin.feedc                 C  r]   rF   r^   r+   r%   r%   r&   r,   ½   rG   z%SuspiciousDuplicateAccentPlugin.resetr-   c                 C  rZ   )Nr   rH   rD   )r<   r_   r+   r%   r%   r&   r.   Â   r[   z%SuspiciousDuplicateAccentPlugin.ratioNr1   r/   r0   r2   rJ   r%   r%   r%   r&   r\   ¥   s    



r\   c                   @  r8   )ÚSuspiciousRanger   r)   c                 C  r]   rF   )Ú"_suspicious_successive_range_countr<   Ú_last_printable_seenr+   r%   r%   r&   r>   Ë   rG   zSuspiciousRange.__init__r   r   r   c                 C  r?   r@   rA   r#   r%   r%   r&   r'   Ð   rC   zSuspiciousRange.eligiblec                 C  sx   |  j d7  _ | ¡ st|ƒs|tv rd | _d S | jd u r"|| _d S t| jƒ}t|ƒ}t||ƒr7|  jd7  _|| _d S rR   )r<   Úisspacer   r   rd   r   Ú is_suspiciously_successive_rangerc   )r$   r   Úunicode_range_aÚunicode_range_br%   r%   r&   r*   Ó   s"   ÿþý



zSuspiciousRange.feedc                 C  r]   rF   )r<   rc   rd   r+   r%   r%   r&   r,   ê   rG   zSuspiciousRange.resetr-   c                 C  s"   | j dkrdS | jd | j  }|S )Né   rH   rD   )r<   rc   )r$   Zratio_of_suspicious_range_usager%   r%   r&   r.   ï   s   
þzSuspiciousRange.ratioNr1   r/   r0   r2   rJ   r%   r%   r%   r&   rb   Ê   s    



rb   c                   @  r8   )ÚSuperWeirdWordPluginr   r)   c                 C  s@   d| _ d| _d| _d| _d| _d| _d| _d| _d| _d| _	d S )Nr   FÚ )
Ú_word_countÚ_bad_word_countÚ_foreign_long_countÚ_is_current_word_badÚ_foreign_long_watchr<   Ú_bad_character_countÚ_bufferÚ_buffer_accent_countÚ_buffer_glyph_countr+   r%   r%   r&   r>   ü   s   
zSuperWeirdWordPlugin.__init__r   r   r   c                 C  rW   rX   r%   r#   r%   r%   r&   r'     rY   zSuperWeirdWordPlugin.eligiblec                 C  sŽ  |  ¡ rc|  j|7  _t|ƒr|  jd7  _| jdu rFt|ƒdu s%t|ƒrFt|ƒdu rFt|ƒdu rFt|ƒdu rFt	|ƒdu rFt
|ƒdu rFd| _t|ƒsZt|ƒsZt|ƒsZt	|ƒsZt
|ƒra|  jd7  _d S | jshd S | ¡ sut|ƒsut|ƒ�r$| j�r$|  jd7  _t| jƒ}|  j|7  _|dkrÏ| j| dkr›d| _n4t| jd ƒrÀ| jd  ¡ rÀtdd„ | jD ƒƒdu rÀ|  jd7  _d| _n| jdkrÏd| _|  jd7  _|d	krþ| jrþd
d„ t| jtd|ƒƒD ƒ}d}|ròt|ƒ| dkròd}|sþ|  jd7  _d| _| j�r|  jd7  _|  jt| jƒ7  _d| _d| _d| _d| _d| _d S |dv�rA| ¡ du �rCt|ƒ�rEd| _|  j|7  _d S d S d S d S )Nr   FTé   ç      à?éÿÿÿÿc                 s  s   � | ]}|  ¡ V  qd S r@   ©ra   )Ú.0Ú_r%   r%   r&   Ú	<genexpr>8  s   € z,SuperWeirdWordPlugin.feed.<locals>.<genexpr>é   c                 S  s   g | ]
\}}|  ¡ r|‘qS r%   rx   )ry   ÚcÚir%   r%   r&   Ú
<listcomp>@  s    ýÿz-SuperWeirdWordPlugin.feed.<locals>.<listcomp>r   rI   rk   >   ú~ú-ú=rz   ú<ú|ú>)rP   rr   r	   rs   rp   r   r   r   r   r   r   rt   re   r   r   rl   Úlenr<   ro   ra   Úallrn   ÚzipÚrangerm   rq   rE   r   )r$   r   Zbuffer_lengthZcamel_case_dstZprobable_camel_casedr%   r%   r&   r*     s®   ÿ
þþ
ý
ü
û
ú
ù	ÿþýüûÿÿÿþ
ÿþý
þ
ÿ
þýúzSuperWeirdWordPlugin.feedc                 C  s4   d| _ d| _d| _d| _d| _d| _d| _d| _d S )Nrk   Fr   )rr   ro   rp   rm   rl   r<   rq   rn   r+   r%   r%   r&   r,   _  s   
zSuperWeirdWordPlugin.resetr-   c                 C  s$   | j dkr| jdkrdS | j| j S )Né
   r   rH   )rl   rn   rq   r<   r+   r%   r%   r&   r.   i  s   zSuperWeirdWordPlugin.ratioNr1   r/   r0   r2   rJ   r%   r%   r%   r&   rj   û   s    



Q
rj   c                   @  sF   e Zd ZdZddd„Zdd	d
„Zddd„Zddd„Zeddd„ƒZ	dS )ÚCjkUncommonPluginz<
    Detect messy CJK text that probably means nothing.
    r   r)   c                 C  rL   rF   ©r<   Ú_uncommon_countr+   r%   r%   r&   r>   v  rO   zCjkUncommonPlugin.__init__r   r   r   c                 C  ó   t |ƒS r@   )r   r#   r%   r%   r&   r'   z  rC   zCjkUncommonPlugin.eligiblec                 C  rQ   rR   )r<   r   r�   r#   r%   r%   r&   r*   }  s
   þzCjkUncommonPlugin.feedc                 C  rL   rF   rŒ   r+   r%   r%   r&   r,   „  rO   zCjkUncommonPlugin.resetr-   c                 C  s.   | j dk rdS | j| j  }|dkr|d S dS )NrT   rH   rv   rŠ   rŒ   )r$   Zuncommon_form_usager%   r%   r&   r.   ˆ  s   
zCjkUncommonPlugin.ratioNr1   r/   r0   r2   )
r3   r4   r5   r6   r>   r'   r*   r,   r7   r.   r%   r%   r%   r&   r‹   q  s    



r‹   c                   @  r8   )ÚArchaicUpperLowerPluginr   r)   c                 C  s.   d| _ d| _d| _d| _d| _d | _d| _d S )NFr   T)Ú_bufÚ_character_count_since_last_sepÚ_successive_upper_lower_countÚ#_successive_upper_lower_count_finalr<   Ú_last_alpha_seenÚ_current_ascii_onlyr+   r%   r%   r&   r>   •  s   
z ArchaicUpperLowerPlugin.__init__r   r   r   c                 C  rW   rX   r%   r#   r%   r%   r&   r'   ¢  rY   z ArchaicUpperLowerPlugin.eligiblec                 C  s$  |  ¡ ot|ƒ}|du }|rC| jdkrC| jdkr+| ¡ du r+| jdu r+|  j| j7  _d| _d| _d | _d| _|  j	d7  _	d| _d S | jdu rQ| 
¡ du rQd| _| jd ur| ¡ r_| j ¡ sh| ¡ r|| j ¡ r|| jdu rx|  jd7  _d| _nd| _nd| _|  j	d7  _	|  jd7  _|| _d S )NFr   é@   r   TrD   )rP   r   r‘   rE   r•   r“   r’   r”   r�   r<   Úisasciira   Úislower)r$   r   Zis_concernedZ	chunk_sepr%   r%   r&   r*   ¥  sF   ÿ
þýÿ
ÿÿ

zArchaicUpperLowerPlugin.feedc                 C  s.   d| _ d| _d| _d| _d | _d| _d| _d S )Nr   FT)r<   r‘   r’   r“   r”   r�   r•   r+   r%   r%   r&   r,   Ï  s   
zArchaicUpperLowerPlugin.resetr-   c                 C  s   | j dkrdS | j| j  S )Nr   rH   )r<   r“   r+   r%   r%   r&   r.   Ø  s   
zArchaicUpperLowerPlugin.ratioNr1   r/   r0   r2   rJ   r%   r%   r%   r&   r�   ”  s    



*	r�   c                   @  sB   e Zd Zddd„Zddd„Zdd
d„Zddd„Zeddd„ƒZdS )ÚArabicIsolatedFormPluginr   r)   c                 C  rL   rF   ©r<   Ú_isolated_form_countr+   r%   r%   r&   r>   á  rO   z!ArabicIsolatedFormPlugin.__init__c                 C  rL   rF   rš   r+   r%   r%   r&   r,   å  rO   zArabicIsolatedFormPlugin.resetr   r   r   c                 C  rŽ   r@   )r
   r#   r%   r%   r&   r'   é  rC   z!ArabicIsolatedFormPlugin.eligiblec                 C  rQ   rR   )r<   r   r›   r#   r%   r%   r&   r*   ì  rS   zArabicIsolatedFormPlugin.feedr-   c                 C  s   | j dk rdS | j| j  }|S )NrT   rH   rš   )r$   Zisolated_form_usager%   r%   r&   r.   ò  s   
zArabicIsolatedFormPlugin.ratioNr1   r/   r0   r2   )	r3   r4   r5   r>   r,   r'   r*   r7   r.   r%   r%   r%   r&   r™   à  s    



r™   é   )Úmaxsizerg   ú
str | Nonerh   r   r   c                 C  sv  | du s|du r
dS | |krdS d| v rd|v rdS d| v s"d|v r$dS d| v s,d|v r6d| v s4d|v r6dS |   d¡|  d¡}}|D ]}|tv rJqC||v rQ dS qC| dv |dv }}|s_|rid	| v sgd	|v ridS |ro|rodS d
| v swd
|v r‹d	| v sd	|v r�dS | dks‰|dkr‹dS d	| v s›d	|v s›| dv r¹|dv r¹d| v s£d|v r¥dS d| v s­d|v r¯dS | dks·|dkr¹dS dS )za
    Determine if two Unicode range seen next to each other can be considered as suspicious.
    NTFZLatinZ	EmoticonsZ	Combiningú )ÚHiraganaÚKatakanaÚCJKZHangulzBasic Latin)r¡   r    ZPunctuationZForms)Úsplitr   )rg   rh   Zkeywords_range_aZkeywords_range_bÚelZrange_a_jp_charsZrange_b_jp_charsr%   r%   r&   rf   ü  sj   ÿÿþÿÿúÿÿÿþrf   i   çš™™™™™É?FÚdecoded_sequencer   Úmaximum_thresholdr-   Údebugc              	   C  sR  dd„ t  ¡ D ƒ}t| ƒd }d}|dk rd}n	|dkrd}nd	}t| d
 t|ƒƒD ]2\}}|D ]}	|	 |¡r<|	 |¡ q0|dkrG|| dksM||d kr\tdd„ |D ƒƒ}||kr\ q]q*|r¤tdƒ}
|
 	t
d|› d|› d|› �¡ t| ƒdkr’|
 	t
d| dd… › �¡ |
 	t
d| dd… › �¡ |D ]}|
 	t
|j› d|j› �¡ q”t|dƒS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    c                 S  s   g | ]}|ƒ ‘qS r%   r%   )ry   Zmd_classr%   r%   r&   r   N  s    ÿzmess_ratio.<locals>.<listcomp>r   rH   i   é    rœ   r–   é€   Ú
r   c                 s  s   � | ]}|j V  qd S r@   )r.   )ry   Údtr%   r%   r&   r{   e  s   € zmess_ratio.<locals>.<genexpr>Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=é   zStarting with: NzEnding with: iðÿÿÿz: é   )r   Ú__subclasses__r†   rˆ   r‰   r'   r*   Úsumr   Úlogr   Ú	__class__r.   Úround)r¦   r§   r¨   Z	detectorsÚlengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcr   ÚindexÚdetectorÚloggerr¬   r%   r%   r&   Ú
mess_ratioF  sV   ÿ

€ÿ
ÿ
þ€ÿÿþþ
r¸   N)rg   rž   rh   rž   r   r   )r¥   F)r¦   r   r§   r-   r¨   r   r   r-   ))Ú
__future__r   Ú	functoolsr   Úloggingr   Zconstantr   r   r   Úutilsr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r9   rK   rU   r\   rb   rj   r‹   r�   r™   rf   r¸   r%   r%   r%   r&   Ú<module>   s(    P"/%1v#LIÿ