U
    U‚gDN  ã                   @  sb  d dl mZ d dlmZ d dlmZ ddlmZmZm	Z	 ddl
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd„ dƒZG d	d
„ d
eƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZ G dd„ deƒZ!G dd„ deƒZ"G dd„ deƒZ#G dd„ deƒZ$G dd„ deƒZ%edd�ddddœd d!„ƒZ&ed"d�d+d%d&dd&d'œd(d)„ƒZ'd*S ),é    )Úannotations)Ú	lru_cache)Ú	getLoggeré   )ÚCOMMON_SAFE_ASCII_CHARACTERSÚTRACEÚUNICODE_SECONDARY_RANGE_KEYWORD)Úis_accentuatedÚ	is_arabicÚis_arabic_isolated_formÚis_case_variableÚis_cjkÚis_emoticonÚ	is_hangulÚis_hiraganaÚis_katakanaÚis_latinÚis_punctuationÚis_separatorÚ	is_symbolÚis_thaiÚis_unprintableÚremove_accentÚunicode_rangec                   @  sP   e Zd ZdZdddœdd„Zdddœdd	„Zdd
œdd„Zedd
œdd„ƒZdS )ÚMessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    ÚstrÚbool©Ú	characterÚreturnc                 C  s   t ‚dS )z@
        Determine if given character should be fed in.
        N©ÚNotImplementedError©Úselfr   © r$   ú9/tmp/pip-unpacked-wheel-5ringpik/charset_normalizer/md.pyÚeligible&   s    zMessDetectorPlugin.eligibleÚNonec                 C  s   t ‚dS )z‰
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        Nr    r"   r$   r$   r%   Úfeed,   s    zMessDetectorPlugin.feed©r   c                 C  s   t ‚dS )zB
        Permit to reset the plugin to the initial state.
        Nr    ©r#   r$   r$   r%   Úreset3   s    zMessDetectorPlugin.resetÚfloatc                 C  s   t ‚dS )z…
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        Nr    r*   r$   r$   r%   Úratio9   s    zMessDetectorPlugin.ratioN)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r&   r(   r+   Úpropertyr-   r$   r$   r$   r%   r       s   r   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )Ú TooManySymbolOrPunctuationPluginr'   r)   c                 C  s"   d| _ d| _d| _d | _d| _d S )Nr   F)Ú_punctuation_countÚ_symbol_countÚ_character_countÚ_last_printable_charZ_frenzy_symbol_in_wordr*   r$   r$   r%   Ú__init__C   s
    z)TooManySymbolOrPunctuationPlugin.__init__r   r   r   c                 C  s   |  ¡ S ©N©Úisprintabler"   r$   r$   r%   r&   K   s    z)TooManySymbolOrPunctuationPlugin.eligiblec                 C  sp   |  j d7  _ || jkrf|tkrft|ƒr8|  jd7  _n.| ¡ dkrft|ƒrft|ƒdkrf|  jd7  _|| _d S )Nr   Fé   )	r6   r7   r   r   r4   Úisdigitr   r   r5   r"   r$   r$   r%   r(   N   s    ÿþ
ÿþ
ýz%TooManySymbolOrPunctuationPlugin.feedc                 C  s   d| _ d| _d| _d S ©Nr   )r4   r6   r5   r*   r$   r$   r%   r+   `   s    z&TooManySymbolOrPunctuationPlugin.resetr,   c                 C  s0   | j dkrdS | j| j | j  }|dkr,|S dS )Nr   ç        ç333333Ó?)r6   r4   r5   )r#   Zratio_of_punctuationr$   r$   r%   r-   e   s    

þz&TooManySymbolOrPunctuationPlugin.ratioN©	r.   r/   r0   r8   r&   r(   r+   r2   r-   r$   r$   r$   r%   r3   B   s   r3   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚTooManyAccentuatedPluginr'   r)   c                 C  s   d| _ d| _d S r>   ©r6   Ú_accentuated_countr*   r$   r$   r%   r8   r   s    z!TooManyAccentuatedPlugin.__init__r   r   r   c                 C  s   |  ¡ S r9   )Úisalphar"   r$   r$   r%   r&   v   s    z!TooManyAccentuatedPlugin.eligiblec                 C  s(   |  j d7  _ t|ƒr$|  jd7  _d S ©Nr   )r6   r	   rD   r"   r$   r$   r%   r(   y   s    zTooManyAccentuatedPlugin.feedc                 C  s   d| _ d| _d S r>   rC   r*   r$   r$   r%   r+      s    zTooManyAccentuatedPlugin.resetr,   c                 C  s*   | j dk rdS | j| j  }|dkr&|S dS )Né   r?   gffffffÖ?rC   )r#   Zratio_of_accentuationr$   r$   r%   r-   ƒ   s    
zTooManyAccentuatedPlugin.ratioNrA   r$   r$   r$   r%   rB   q   s   rB   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚUnprintablePluginr'   r)   c                 C  s   d| _ d| _d S r>   )Ú_unprintable_countr6   r*   r$   r$   r%   r8   �   s    zUnprintablePlugin.__init__r   r   r   c                 C  s   dS ©NTr$   r"   r$   r$   r%   r&   ‘   s    zUnprintablePlugin.eligiblec                 C  s(   t |ƒr|  jd7  _|  jd7  _d S rF   )r   rI   r6   r"   r$   r$   r%   r(   ”   s    zUnprintablePlugin.feedc                 C  s
   d| _ d S r>   )rI   r*   r$   r$   r%   r+   ™   s    zUnprintablePlugin.resetr,   c                 C  s   | j dkrdS | jd | j  S )Nr   r?   rG   )r6   rI   r*   r$   r$   r%   r-   œ   s    
zUnprintablePlugin.ratioNrA   r$   r$   r$   r%   rH   Œ   s   rH   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚSuspiciousDuplicateAccentPluginr'   r)   c                 C  s   d| _ d| _d | _d S r>   ©Ú_successive_countr6   Ú_last_latin_characterr*   r$   r$   r%   r8   ¥   s    z(SuspiciousDuplicateAccentPlugin.__init__r   r   r   c                 C  s   |  ¡ ot|ƒS r9   )rE   r   r"   r$   r$   r%   r&   «   s    z(SuspiciousDuplicateAccentPlugin.eligiblec                 C  st   |  j d7  _ | jd k	rjt|ƒrjt| jƒrj| ¡ rJ| j ¡ rJ|  jd7  _t|ƒt| jƒkrj|  jd7  _|| _d S rF   )r6   rN   r	   ÚisupperrM   r   r"   r$   r$   r%   r(   ®   s    ÿþýz$SuspiciousDuplicateAccentPlugin.feedc                 C  s   d| _ d| _d | _d S r>   rL   r*   r$   r$   r%   r+   ¼   s    z%SuspiciousDuplicateAccentPlugin.resetr,   c                 C  s   | j dkrdS | jd | j  S )Nr   r?   r<   )r6   rM   r*   r$   r$   r%   r-   Á   s    
z%SuspiciousDuplicateAccentPlugin.ratioNrA   r$   r$   r$   r%   rK   ¤   s   rK   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚSuspiciousRanger'   r)   c                 C  s   d| _ d| _d | _d S r>   )Ú"_suspicious_successive_range_countr6   Ú_last_printable_seenr*   r$   r$   r%   r8   Ê   s    zSuspiciousRange.__init__r   r   r   c                 C  s   |  ¡ S r9   r:   r"   r$   r$   r%   r&   Ï   s    zSuspiciousRange.eligiblec                 C  sx   |  j d7  _ | ¡ s&t|ƒs&|tkr0d | _d S | jd krD|| _d S t| jƒ}t|ƒ}t||ƒrn|  jd7  _|| _d S rF   )r6   Úisspacer   r   rR   r   Ú is_suspiciously_successive_rangerQ   )r#   r   Úunicode_range_aÚunicode_range_br$   r$   r%   r(   Ò   s"    ÿþý


zSuspiciousRange.feedc                 C  s   d| _ d| _d | _d S r>   )r6   rQ   rR   r*   r$   r$   r%   r+   é   s    zSuspiciousRange.resetr,   c                 C  s"   | j dkrdS | jd | j  }|S )Né   r?   r<   )r6   rQ   )r#   Zratio_of_suspicious_range_usager$   r$   r%   r-   î   s    
þzSuspiciousRange.ratioNrA   r$   r$   r$   r%   rP   É   s   rP   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚSuperWeirdWordPluginr'   r)   c                 C  s@   d| _ d| _d| _d| _d| _d| _d| _d| _d| _d| _	d S )Nr   FÚ )
Ú_word_countÚ_bad_word_countÚ_foreign_long_countÚ_is_current_word_badÚ_foreign_long_watchr6   Ú_bad_character_countÚ_bufferÚ_buffer_accent_countÚ_buffer_glyph_countr*   r$   r$   r%   r8   û   s    zSuperWeirdWordPlugin.__init__r   r   r   c                 C  s   dS rJ   r$   r"   r$   r$   r%   r&   
  s    zSuperWeirdWordPlugin.eligiblec                 C  s–  |  ¡ rÆ|  j|7  _t|ƒr,|  jd7  _| jdkrŒt|ƒdksJt|ƒrŒt|ƒdkrŒt|ƒdkrŒt|ƒdkrŒt	|ƒdkrŒt
|ƒdkrŒd| _t|ƒs´t|ƒs´t|ƒs´t	|ƒs´t
|ƒrÂ|  jd7  _d S | jsÐd S | ¡ sêt|ƒsêt|ƒ�r\| j�r\|  jd7  _t| jƒ}|  j|7  _|dk�rª| j| dk�r:d| _npt| jd ƒ�rŠ| jd  ¡ �rŠtdd„ | jD ƒƒdk�rŠ|  jd7  _d| _n | jdk�rªd| _|  jd7  _|d	k�r| j�rd
d„ t| jtd|ƒƒD ƒ}d}|�røt|ƒ| dk�rød}|�s|  jd7  _d| _| j�rB|  jd7  _|  jt| jƒ7  _d| _d| _d| _d| _d| _n6|dk�r’| ¡ dk�r’t|ƒ�r’d| _|  j|7  _d S )Nr   FTé   g      à?éÿÿÿÿc                 s  s   | ]}|  ¡ V  qd S r9   ©rO   )Ú.0Ú_r$   r$   r%   Ú	<genexpr>7  s     z,SuperWeirdWordPlugin.feed.<locals>.<genexpr>é   c                 S  s   g | ]\}}|  ¡ r|‘qS r$   re   )rf   ÚcÚir$   r$   r%   Ú
<listcomp>?  s   þz-SuperWeirdWordPlugin.feed.<locals>.<listcomp>r   r@   rY   >   ú=ú>ú|ú-ú~ú<rg   )rE   r`   r	   ra   r^   r   r   r   r   r   r   rb   rS   r   r   rZ   Úlenr6   r]   rO   Úallr\   ÚzipÚranger[   r_   r=   r   )r#   r   Úbuffer_lengthZcamel_case_dstZprobable_camel_casedr$   r$   r%   r(     s¬    ÿ
þþ
ý
ü
û
ú
ù	ÿþýüûÿÿÿþ

ÿþýþÿ
þýzSuperWeirdWordPlugin.feedc                 C  s4   d| _ d| _d| _d| _d| _d| _d| _d| _d S )NrY   Fr   )r`   r]   r^   r[   rZ   r6   r_   r\   r*   r$   r$   r%   r+   ^  s    zSuperWeirdWordPlugin.resetr,   c                 C  s$   | j dkr| jdkrdS | j| j S )Né
   r   r?   )rZ   r\   r_   r6   r*   r$   r$   r%   r-   h  s    zSuperWeirdWordPlugin.ratioNrA   r$   r$   r$   r%   rX   ú   s   Q
rX   c                   @  s^   e Zd ZdZddœdd„Zdddœd	d
„Zdddœdd„Zddœdd„Zeddœdd„ƒZ	dS )ÚCjkInvalidStopPluginu²   
    GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and
    can be easily detected. Searching for the overuse of 'ä¸…' and 'ä¸„'.
    r'   r)   c                 C  s   d| _ d| _d S r>   ©Ú_wrong_stop_countÚ_cjk_character_countr*   r$   r$   r%   r8   v  s    zCjkInvalidStopPlugin.__init__r   r   r   c                 C  s   dS rJ   r$   r"   r$   r$   r%   r&   z  s    zCjkInvalidStopPlugin.eligiblec                 C  s4   |dkr|  j d7  _ d S t|ƒr0|  jd7  _d S )N>   õ   ä¸…õ   ä¸„r   )r{   r   r|   r"   r$   r$   r%   r(   }  s
    zCjkInvalidStopPlugin.feedc                 C  s   d| _ d| _d S r>   rz   r*   r$   r$   r%   r+   „  s    zCjkInvalidStopPlugin.resetr,   c                 C  s   | j dk rdS | j| j  S )Né   r?   )r|   r{   r*   r$   r$   r%   r-   ˆ  s    
zCjkInvalidStopPlugin.ratioN)
r.   r/   r0   r1   r8   r&   r(   r+   r2   r-   r$   r$   r$   r%   ry   p  s   ry   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚArchaicUpperLowerPluginr'   r)   c                 C  s.   d| _ d| _d| _d| _d| _d | _d| _d S )NFr   T)Ú_bufÚ_character_count_since_last_sepÚ_successive_upper_lower_countÚ#_successive_upper_lower_count_finalr6   Ú_last_alpha_seenÚ_current_ascii_onlyr*   r$   r$   r%   r8   �  s    z ArchaicUpperLowerPlugin.__init__r   r   r   c                 C  s   dS rJ   r$   r"   r$   r$   r%   r&   �  s    z ArchaicUpperLowerPlugin.eligiblec                 C  s$  |  ¡ ot|ƒ}|dk}|r†| jdkr†| jdkrV| ¡ dkrV| jdkrV|  j| j7  _d| _d| _d | _d| _|  j	d7  _	d| _d S | jdkr¢| 
¡ dkr¢d| _| jd k	rþ| ¡ r¾| j ¡ sÐ| ¡ rø| j ¡ rø| jdkrð|  jd7  _d| _qþd| _nd| _|  j	d7  _	|  jd7  _|| _d S )NFr   é@   r   Tr<   )rE   r   r‚   r=   r†   r„   rƒ   r…   r�   r6   ÚisasciirO   Úislower)r#   r   Zis_concernedZ	chunk_sepr$   r$   r%   r(      sF    ÿ
þýÿ
ÿÿ
zArchaicUpperLowerPlugin.feedc                 C  s.   d| _ d| _d| _d| _d | _d| _d| _d S )Nr   FT)r6   r‚   rƒ   r„   r…   r�   r†   r*   r$   r$   r%   r+   Ê  s    zArchaicUpperLowerPlugin.resetr,   c                 C  s   | j dkrdS | j| j  S )Nr   r?   )r6   r„   r*   r$   r$   r%   r-   Ó  s    
zArchaicUpperLowerPlugin.ratioNrA   r$   r$   r$   r%   r€   �  s   *	r€   c                   @  sZ   e Zd Zddœdd„Zddœdd„Zddd	œd
d„Zddd	œdd„Zeddœdd„ƒZdS )ÚArabicIsolatedFormPluginr'   r)   c                 C  s   d| _ d| _d S r>   ©r6   Ú_isolated_form_countr*   r$   r$   r%   r8   Ü  s    z!ArabicIsolatedFormPlugin.__init__c                 C  s   d| _ d| _d S r>   r‹   r*   r$   r$   r%   r+   à  s    zArabicIsolatedFormPlugin.resetr   r   r   c                 C  s   t |ƒS r9   )r
   r"   r$   r$   r%   r&   ä  s    z!ArabicIsolatedFormPlugin.eligiblec                 C  s(   |  j d7  _ t|ƒr$|  jd7  _d S rF   )r6   r   rŒ   r"   r$   r$   r%   r(   ç  s    zArabicIsolatedFormPlugin.feedr,   c                 C  s   | j dk rdS | j| j  }|S )NrG   r?   r‹   )r#   Zisolated_form_usager$   r$   r%   r-   í  s    
zArabicIsolatedFormPlugin.ratioN)	r.   r/   r0   r8   r+   r&   r(   r2   r-   r$   r$   r$   r%   rŠ   Û  s   rŠ   é   )Úmaxsizez
str | Noner   )rU   rV   r   c                 C  s”  | dks|dkrdS | |kr dS d| kr4d|kr4dS d| ksDd|krHdS d| ksXd|krld| kshd|krldS |   d¡|  d¡ }}|D ]}|tkr”q†||kr† dS q†| dk|dk }}|s¾|rÒd	| ksÎd	|krÒdS |rÞ|rÞdS d
| ksðd
|k�r d	| k�sd	|k�rdS | dk�s|dk�r dS d	| k�sHd	|k�sH| dk�r�|dk�r�d| k�s\d|k�r`dS d| k�std|k�rxdS | dk�sŒ|dk�r�dS dS )za
    Determine if two Unicode range seen next to each other can be considered as suspicious.
    NTFZLatinZ	EmoticonsZ	Combiningú )ÚHiraganaÚKatakanaÚCJKZHangulzBasic Latin)r‘   r�   ZPunctuationZForms)Úsplitr   )rU   rV   Zkeywords_range_aZkeywords_range_bÚelZrange_a_jp_charsZrange_b_jp_charsr$   r$   r%   rT   ÷  sh    ÿÿþÿúÿÿÿþrT   i   çš™™™™™É?Fr   r,   )Údecoded_sequenceÚmaximum_thresholdÚdebugr   c              	   C  sX  dd„ t  ¡ D ƒ}t| ƒd }d}|dk r0d}n|dkr>d}nd	}t| d
 t|ƒƒD ]d\}}|D ]}	|	 |¡r`|	 |¡ q`|dkrŽ|| dksš||d krTtdd„ |D ƒƒ}||krT qºqT|�rNtdƒ}
|
 	t
d|› d|› d|› �¡ t| ƒdk�r(|
 	t
d| dd… › �¡ |
 	t
d| dd… › �¡ |D ] }|
 	t
|j› d|j› �¡ �q,t|dƒS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    c                 S  s   g | ]
}|ƒ ‘qS r$   r$   )rf   Zmd_classr$   r$   r%   rl   I  s    zmess_ratio.<locals>.<listcomp>r   r?   i   é    r�   r‡   é€   Ú
r   c                 s  s   | ]}|j V  qd S r9   )r-   )rf   Údtr$   r$   r%   rh   `  s     zmess_ratio.<locals>.<genexpr>Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=r   zStarting with: NzEnding with: iðÿÿÿz: é   )r   Ú__subclasses__rs   ru   rv   r&   r(   Úsumr   Úlogr   Ú	__class__r-   Úround)r–   r—   r˜   Z	detectorsÚlengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcr   ÚindexÚdetectorÚloggerrœ   r$   r$   r%   Ú
mess_ratioA  sF    ÿ
ÿ
ÿ
þþr§   N)r•   F)(Ú
__future__r   Ú	functoolsr   Úloggingr   Zconstantr   r   r   Úutilsr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r3   rB   rH   rK   rP   rX   ry   r€   rŠ   rT   r§   r$   r$   r$   r%   Ú<module>   s(   L"/%1vLI   ÿ