U
    ©Acj�  ã                   @  s.  U d dl mZ d dlZd dlmZ d dlmZ ejdkrFd dlm	Z	 n.zd dl
m	Z	 W n ek
rr   dd„ Z	Y nX d	d
lmZmZmZmZmZmZmZmZmZmZmZmZmZ d	dlmZmZmZmZmZm Z m!Z! eeB eB eB eB Z"de#d< e	G dd„ dƒƒZ$edd�dddœdd„ƒZ%dd„ e&dƒD ƒZ'de#d< G dd„ dƒZ(e	G dd„ de(ƒƒZ)e	G dd„ de(ƒƒZ*e	G d d!„ d!e(ƒƒZ+e	G d"d#„ d#e(ƒƒZ,e	G d$d%„ d%e(ƒƒZ-e	G d&d'„ d'e(ƒƒZ.e	G d(d)„ d)e(ƒƒZ/e	G d*d+„ d+e(ƒƒZ0e	G d,d-„ d-e(ƒƒZ1ed.d�d/d/d0d1œd2d3„ƒZ2d:dd6d0d6d7œd8d9„Z3dS );é    )ÚannotationsN)Ú	lru_cache)Ú	getLogger)é   é   )Úfinalc                 C  s   | S )N© )Úclsr   r   ú9/tmp/pip-unpacked-wheel-k0k9q145/charset_normalizer/md.pyr      s    r   é   )ÚCOMMON_CJK_CHARACTERSÚCOMMON_SAFE_ASCII_CHARACTERSÚTRACEÚUNICODE_SECONDARY_RANGE_KEYWORDÚ_ACCENTUATEDÚ_ARABICÚ_ARABIC_ISOLATED_FORMÚ_CJKÚ_HANGULÚ	_HIRAGANAÚ	_KATAKANAÚ_LATINÚ_THAI)Ú_character_flagsÚis_emoticonÚis_punctuationÚis_separatorÚ	is_symbolÚremove_accentÚunicode_rangeÚintÚ_GLYPH_MASKc                   @  sÖ   e Zd ZU dZdZded< ded< ded< ded< ded	< ded
< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< dddœdd „Zd!S )"ÚCharInfoz>Pre-computed character properties shared across all detectors.)Ú	characterÚ	printableÚalphaÚupperÚlowerÚspaceÚdigitÚis_asciiÚcase_variableÚflagsÚaccentuatedÚlatinÚis_cjkÚ	is_arabicÚis_glyphÚpunctÚsymÚrangeÚsepÚemoticonÚsafeÚ
common_cjkÚstrr#   Úboolr$   r%   r&   r'   r(   r)   r*   r+   r    r,   r-   r.   r/   r0   r1   r2   r3   ú
str | Noner4   r5   r6   r7   r8   ÚNone©r#   Úreturnc                 C  s<  || _ t|ƒ}|dk �r(d| _d| _d| _d| _d| _d|  krJdkr”n nFd| _d| _d| _	d| _
d| _d| _d| _t| _d| _d| _d| _�qd|  kr¨dkròn nFd| _d| _d| _	d| _
d| _d| _d| _t| _d| _d| _d| _�qd|  k�r
d	k�rRn nDd| _d| _d| _	d| _
d| _d| _d| _d
| _d| _d| _d| _nÔ|dk�sxd|  k�rtdk�rÀn nHd| _d| _d| _	d| _
d| _|dk| _d| _d
| _d| _d| _d| _nf| ¡ | _d| _d| _d| _	d| _
d| _d| _d
| _d| _| j�r
t|ƒnd| _| j�r t|ƒnd| _nÞd| _| ¡ | _| ¡ | _| ¡ | _| ¡ | _	| ¡ | _
| ¡ | _| j	| jk| _| j�rŠt|ƒ}nd
}|| _t|t@ ƒ| _t|t@ ƒ| _t|t@ ƒ| _t|t@ ƒ| _t|t @ ƒ| _| j�rêt|ƒnd| _| j�r t|ƒnd| _t!|ƒ| _"t#|ƒ| _$t%|ƒ| _&|t'k| _(|t)k| _*dS )zkCompute all properties for *character* (built once per codepoint,
        every branch assigns every slot).é€   TFéA   éZ   éa   éz   é0   é9   r   é    é	   é   N)+r#   Úordr*   r-   r/   r0   r1   r%   r&   r'   r(   r)   r$   r+   r   r,   r.   r2   r3   Úisprintabler   r   ÚisalphaÚisupperÚislowerÚisspaceÚisdigitr   r:   r   r   r   r!   r   r4   r   r5   r   r6   r   r7   r   r8   )Úselfr#   Úor,   r   r   r
   Ú__init__c   s¶    


&












zCharInfo.__init__N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú	__slots__Ú__annotations__rR   r   r   r   r
   r"   /   s2   
r"   )Úmaxsizer9   r=   c                 C  s   t | ƒS )zBBuild (once per codepoint) and cache the CharInfo for *character*.)r"   )r#   r   r   r
   Ú
_char_infoÙ   s    rZ   c                 C  s   g | ]}t t|ƒƒ‘qS r   )r"   Úchr)Ú.0Z
_codepointr   r   r
   Ú
<listcomp>à   s    r]   r?   zlist[CharInfo]Ú_ASCII_CHAR_INFOc                   @  sF   e Zd ZdZdZddddœdd„Zdd	œd
d„Zedd	œdd„ƒZdS )ÚMessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    r   r9   r"   r<   ©r#   Úinfor>   c                 C  s   t ‚dS )z‰
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        N©ÚNotImplementedError©rP   r#   ra   r   r   r
   Ú	feed_infoí   s    zMessDetectorPlugin.feed_info©r>   c                 C  s   t ‚dS )zB
        Permit to reset the plugin to the initial state.
        Nrb   ©rP   r   r   r
   Úresetô   s    zMessDetectorPlugin.resetÚfloatc                 C  s   t ‚dS )z…
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        Nrb   rg   r   r   r
   Úratioú   s    zMessDetectorPlugin.ratioN)	rS   rT   rU   rV   rW   re   rh   Úpropertyrj   r   r   r   r
   r_   å   s   r_   c                   @  sP   e Zd ZdZddœdd„Zddddœd	d
„Zddœdd„Zeddœdd„ƒZdS )Ú TooManySymbolOrPunctuationPlugin©Ú_punctuation_countÚ_symbol_countÚ_character_countÚ_last_printable_charZ_frenzy_symbol_in_wordr<   rf   c                 C  s"   d| _ d| _d| _d | _d| _d S ©Nr   Frm   rg   r   r   r
   rR     s
    z)TooManySymbolOrPunctuationPlugin.__init__r9   r"   r`   c                 C  s^   |  j d7  _ || jkrT|jsT|jr4|  jd7  _n |jsT|jrT|jsT|  jd7  _|| _dS )ú1Optimized feed using pre-computed character info.r   é   N)	rp   rq   r7   r2   rn   r)   r3   r6   ro   rd   r   r   r
   re     s    z*TooManySymbolOrPunctuationPlugin.feed_infoc                 C  s   d| _ d| _d| _d S ©Nr   )rn   rp   ro   rg   r   r   r
   rh   !  s    z&TooManySymbolOrPunctuationPlugin.resetri   c                 C  s0   | j dkrdS | j| j | j  }|dkr,|S dS )Nr   ç        ç333333Ó?)rp   rn   ro   )rP   Zratio_of_punctuationr   r   r
   rj   &  s    

þz&TooManySymbolOrPunctuationPlugin.ratioN©	rS   rT   rU   rW   rR   re   rh   rk   rj   r   r   r   r
   rl     s   rl   c                   @  sP   e Zd ZdZddœdd„Zddddœd	d
„Zddœdd„Zeddœdd„ƒZdS )ÚTooManyAccentuatedPlugin©rp   Ú_accentuated_countr<   rf   c                 C  s   d| _ d| _d S ru   rz   rg   r   r   r
   rR   6  s    z!TooManyAccentuatedPlugin.__init__r9   r"   r`   c                 C  s&   |  j d7  _ |jr"|  jd7  _dS ©rs   r   N)rp   r-   r{   rd   r   r   r
   re   :  s    z"TooManyAccentuatedPlugin.feed_infoc                 C  s   d| _ d| _d S ru   rz   rg   r   r   r
   rh   A  s    zTooManyAccentuatedPlugin.resetri   c                 C  s*   | j dk rdS | j| j  }|dkr&|S dS )Nr   rv   gffffffÖ?rz   )rP   Zratio_of_accentuationr   r   r
   rj   E  s    
zTooManyAccentuatedPlugin.ratioNrx   r   r   r   r
   ry   2  s   ry   c                   @  sP   e Zd ZdZddœdd„Zddddœd	d
„Zddœdd„Zeddœdd„ƒZdS )ÚUnprintablePlugin©Ú_unprintable_countrp   r<   rf   c                 C  s   d| _ d| _d S ru   r~   rg   r   r   r
   rR   R  s    zUnprintablePlugin.__init__r9   r"   r`   c                 C  s<   |j s*|js*|dkr*|dkr*|  jd7  _|  jd7  _dS )rs   úu   ï»¿r   N)r(   r$   r   rp   rd   r   r   r
   re   V  s    ÿþýüzUnprintablePlugin.feed_infoc                 C  s
   d| _ d S ru   )r   rg   r   r   r
   rh   a  s    zUnprintablePlugin.resetri   c                 C  s   | j dkrdS | jd | j  S )Nr   rv   r   )rp   r   rg   r   r   r
   rj   d  s    
zUnprintablePlugin.ratioNrx   r   r   r   r
   r}   N  s   r}   c                   @  sP   e Zd ZdZddœdd„Zddddœd	d
„Zddœdd„Zeddœdd„ƒZdS )ÚSuspiciousDuplicateAccentPlugin©Ú_successive_countrp   Ú_last_latin_characterÚ_last_was_accentuatedr<   rf   c                 C  s   d| _ d| _d | _d| _d S rr   r‚   rg   r   r   r
   rR   u  s    z(SuspiciousDuplicateAccentPlugin.__init__r9   r"   r`   c                 C  st   |  j d7  _ | jdk	rb|jrb| jrb|jrB| j ¡ rB|  jd7  _t|ƒt| jƒkrb|  jd7  _|| _|j| _dS r|   )rp   r„   r-   r…   r&   rL   rƒ   r   rd   r   r   r
   re   |  s    ÿþýz)SuspiciousDuplicateAccentPlugin.feed_infoc                 C  s   d| _ d| _d | _d| _d S rr   r‚   rg   r   r   r
   rh   ‹  s    z%SuspiciousDuplicateAccentPlugin.resetri   c                 C  s   | j dkrdS | jd | j  S )Nr   rv   rt   )rp   rƒ   rg   r   r   r
   rj   ‘  s    
z%SuspiciousDuplicateAccentPlugin.ratioNrx   r   r   r   r
   r�   l  s   r�   c                   @  sP   e Zd ZdZddœdd„Zddddœd	d
„Zddœdd„Zeddœdd„ƒZdS )ÚSuspiciousRange©Ú"_suspicious_successive_range_countrp   Ú_last_printable_seenÚ_last_printable_ranger<   rf   c                 C  s   d| _ d| _d | _d | _d S ru   r‡   rg   r   r   r
   rR   ¢  s    zSuspiciousRange.__init__r9   r"   r`   c                 C  s�   |  j d7  _ |js |js |jr0d| _d| _dS | jdkrL|| _|j| _dS | j}|j}||ksh|dkr€t||ƒr€|  jd7  _|| _|| _dS r|   )	rp   r(   r2   r7   r‰   rŠ   r4   Ú is_suspiciously_successive_rangerˆ   )rP   r#   ra   Úunicode_range_aÚunicode_range_br   r   r
   re   ¨  s     

zSuspiciousRange.feed_infoc                 C  s   d| _ d| _d | _d | _d S ru   )rp   rˆ   r‰   rŠ   rg   r   r   r
   rh   Á  s    zSuspiciousRange.resetri   c                 C  s"   | j dkrdS | jd | j  }|S )NrH   rv   rt   )rp   rˆ   )rP   Zratio_of_suspicious_range_usager   r   r
   rj   Ç  s    
þzSuspiciousRange.ratioNrx   r   r   r   r
   r†   ™  s   r†   c                   @  sP   e Zd ZdZddœdd„Zddddœd	d
„Zddœdd„Zeddœdd„ƒZdS )ÚSuperWeirdWordPlugin©Ú_word_countÚ_bad_word_countÚ_foreign_long_countÚ_is_current_word_badÚ_foreign_long_watchrp   Ú_bad_character_countÚ_buffer_lengthÚ_buffer_last_charÚ_buffer_last_char_accentuatedÚ_buffer_accent_countÚ_buffer_glyph_countÚ_buffer_upper_countÚ_buffer_first_lowerÚ_buffer_has_non_asciir<   rf   c                 C  s^   d| _ d| _d| _d| _d| _d| _d| _d| _d | _d| _	d| _
d| _d| _d| _d| _d S rr   r�   rg   r   r   r
   rR   ç  s    zSuperWeirdWordPlugin.__init__r9   r"   r`   c                 C  sp  |j rž| jdkr|j| _|  jd7  _|| _|jr@|  jd7  _|jsLd| _|j	| _
|j	rh|  jd7  _| js†|jrz|j	r†|js†d| _|jrš|  jd7  _dS | js¨dS |js¼|js¼|j�r2|  jd7  _| j}|  j|7  _|dk�rŠ| j| dkrþd| _nŒ| j
�r4| j ¡ �r4| j|k�r4|  jd7  _d| _nV| jdk�rVd| _|  jd7  _n4| j�rŠ| j�rŠ| j|d k�rŠ|  jd7  _d| _|dk�rÐ| j�rÐ| jdk�o´| j| dk}|�sÐ|  jd7  _d| _| j�rú|  jd7  _|  j|7  _d	| _d	| _d| _d| _d	| _
d| _d| _d| _d	| _d	| _n:|d
k�rl|j�sl|j�rld| _|  jd7  _|| _d	| _
dS )rs   r   r   TNé   ç      à?é   rw   F>   ú>ú=ú<ú|ú~Ú_ú-)r%   r–   r'   rœ   r—   r&   r›   r*   r�   r-   r˜   r™   r”   r.   r1   rš   r(   r2   r5   r�   rp   r“   rL   r’   r‘   r•   r)   r3   )rP   r#   ra   Úbuffer_lengthZprobable_camel_casedr   r   r
   re   û  s¤    
ÿþþý
ÿþýÿþýþÿþýzSuperWeirdWordPlugin.feed_infoc                 C  s^   d| _ d | _d| _d| _d| _d| _d| _d| _d| _d| _	d| _
d| _d| _d| _d| _d S rr   )r–   r—   r˜   r“   r”   r‘   r�   rp   r•   r’   r™   rš   r›   rœ   r�   rg   r   r   r
   rh   V  s    zSuperWeirdWordPlugin.resetri   c                 C  s$   | j dkr| jdkrdS | j| j S )Né
   r   rv   )r�   r’   r•   rp   rg   r   r   r
   rj   g  s    zSuperWeirdWordPlugin.ratioNrx   r   r   r   r
   rŽ   Ó  s   [rŽ   c                   @  sT   e Zd ZdZdZddœdd„Zdddd	œd
d„Zddœdd„Zeddœdd„ƒZ	dS )ÚCjkUncommonPluginz<
    Detect messy CJK text that probably means nothing.
    ©rp   Ú_uncommon_countr<   rf   c                 C  s   d| _ d| _d S ru   r«   rg   r   r   r
   rR   w  s    zCjkUncommonPlugin.__init__r9   r"   r`   c                 C  s&   |  j d7  _ |js"|  jd7  _dS r|   )rp   r8   r¬   rd   r   r   r
   re   {  s    zCjkUncommonPlugin.feed_infoc                 C  s   d| _ d| _d S ru   r«   rg   r   r   r
   rh   ‚  s    zCjkUncommonPlugin.resetri   c                 C  s.   | j dk rdS | j| j  }|dkr*|d S dS )Nr   rv   rŸ   r©   r«   )rP   Zuncommon_form_usager   r   r
   rj   †  s    
zCjkUncommonPlugin.ratioN)
rS   rT   rU   rV   rW   rR   re   rh   rk   rj   r   r   r   r
   rª   o  s   rª   c                   @  sP   e Zd ZdZddœdd„Zddddœd	d
„Zddœdd„Zeddœdd„ƒZdS )ÚArchaicUpperLowerPlugin©	Ú_bufÚ_character_count_since_last_sepÚ_successive_upper_lower_countÚ#_successive_upper_lower_count_finalrp   Ú_last_alpha_seenÚ_last_alpha_seen_upperÚ_last_alpha_seen_lowerÚ_current_ascii_onlyr<   rf   c                 C  s:   d| _ d| _d| _d| _d| _d | _d| _d| _d| _d S )NFr   Tr®   rg   r   r   r
   rR      s    z ArchaicUpperLowerPlugin.__init__r9   r"   r`   c                 C  s
  |j o
|j}| }|rv| jdkrv| jdkrF|jsF| jsF|  j| j7  _d| _d| _d| _d| _|  j	d7  _	d| _dS | jrˆ|j
sˆd| _| jdk	rÔ|jrž| jsª|jrÎ| jrÎ| jrÆ|  jd7  _d| _qÔd| _nd| _|  j	d7  _	|  jd7  _|| _|j| _|j| _dS )rs   r   é@   NFr   Trt   )r%   r+   r°   r)   r¶   r²   r±   r³   r¯   rp   r*   r&   rµ   r'   r´   )rP   r#   ra   Zis_concernedZ	chunk_sepr   r   r
   re   ¯  sJ    ÿþýÿ
ÿÿz!ArchaicUpperLowerPlugin.feed_infoc                 C  s:   d| _ d| _d| _d| _d | _d| _d| _d| _d| _d S )Nr   FT)	rp   r°   r±   r²   r³   r´   rµ   r¯   r¶   rg   r   r   r
   rh   Ü  s    zArchaicUpperLowerPlugin.resetri   c                 C  s   | j dkrdS | j| j  S )Nr   rv   )rp   r²   rg   r   r   r
   rj   ç  s    
zArchaicUpperLowerPlugin.ratioNrx   r   r   r   r
   r­   ’  s   -r­   c                   @  sP   e Zd ZdZddœdd„Zddœdd„Zdd	dd
œdd„Zeddœdd„ƒZdS )ÚArabicIsolatedFormPlugin©rp   Ú_isolated_form_countr<   rf   c                 C  s   d| _ d| _d S ru   r¹   rg   r   r   r
   rR   ó  s    z!ArabicIsolatedFormPlugin.__init__c                 C  s   d| _ d| _d S ru   r¹   rg   r   r   r
   rh   ÷  s    zArabicIsolatedFormPlugin.resetr9   r"   r`   c                 C  s*   |  j d7  _ |jt@ r&|  jd7  _dS r|   )rp   r,   r   rº   rd   r   r   r
   re   û  s    
z"ArabicIsolatedFormPlugin.feed_infori   c                 C  s   | j dk rdS | j| j  }|S )Nr   rv   r¹   )rP   Zisolated_form_usager   r   r
   rj     s    
zArabicIsolatedFormPlugin.ratioN)	rS   rT   rU   rW   rR   rh   re   rk   rj   r   r   r   r
   r¸   ï  s   r¸   é   r;   r:   )rŒ   r�   r>   c                 C  s”  | dks|dkrdS | |kr dS d| kr4d|kr4dS d| ksDd|krHdS d| ksXd|krld| kshd|krldS |   d¡|  d¡ }}|D ]}|tkr”q†||kr† dS q†| dk|dk }}|s¾|rÒd	| ksÎd	|krÒdS |rÞ|rÞdS d
| ksðd
|k�r d	| k�sd	|k�rdS | dk�s|dk�r dS d	| k�sHd	|k�sH| dk�r�|dk�r�d| k�s\d|k�r`dS d| k�std|k�rxdS | dk�sŒ|dk�r�dS dS )za
    Determine if two Unicode range seen next to each other can be considered as suspicious.
    NTFZLatinZ	EmoticonsZ	Combiningú )ÚHiraganaÚKatakanaÚCJKZHangulzBasic Latin)r¾   r½   ZPunctuationZForms)Úsplitr   )rŒ   r�   Zkeywords_range_aZkeywords_range_bÚelZrange_a_jp_charsZrange_b_jp_charsr   r   r
   r‹     sh    ÿÿþÿúÿÿÿþr‹   çš™™™™™É?Fri   )Údecoded_sequenceÚmaximum_thresholdÚdebugr>   c           "   	   C  sØ  t | ƒ}|dk rd}n|dk r$d}nd}|  ¡ }t}t}tƒ }tƒ }	tƒ }
tƒ }tƒ }t	ƒ }t
ƒ }tƒ }tƒ }|j}|	j}|
j}|j}|j}|j}|j}|j}|j}td||ƒD �]}| ||| … D ]¼}t|ƒ}|dk rä|| }n||ƒ}|||ƒ |||ƒ |�r|jrÆ|||ƒ qÆ|||ƒ |j�r>|||ƒ |||ƒ |jrÆ|||ƒ |j�r`|||ƒ |j�rr|||ƒ |jrÆ|||ƒ qÆ|j|	j |
j |j |j |j |j |j |j }||kr° �q,q°|d }|d|ƒ |�sì|d|ƒ |d|ƒ |j|	j |
j |j |j |j |j |j |j }|�rÎtd	ƒ} |  td
|› d|› d|› �¡ |dk�r–|  td| dd… › �¡ |  td| dd… › �¡ ||	|
||||||f	D ] }!|  t|!j› d|!j› �¡ �q¬t|dƒS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    iÿ  rF   r»   r·   r?   r   r©   Ú
Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=é   zStarting with: NzEnding with: iðÿÿÿz: r   )ÚlenÚisasciir^   rZ   rl   ry   r}   r�   r†   rŽ   rª   r­   r¸   re   r4   rI   r$   r%   r.   r/   r0   rj   r   Úlogr   Ú	__class__Úround)"rÃ   rÄ   rÅ   Zseq_lenÚstepZis_pure_asciiZ
ascii_infoZ	char_infoZd_spZd_taZd_upZd_sdaZd_srZd_swZd_cuZd_auZd_aiZ	d_sp_feedZ	d_ta_feedZ	d_up_feedZ
d_sda_feedZ	d_sr_feedZ	d_sw_feedZ	d_cu_feedZ	d_au_feedZ	d_ai_feedZblock_startr#   Ú	codepointra   Zmean_mess_ratioZnl_infoÚloggerÚdtr   r   r
   Ú
mess_ratioV  sÔ    	









ÿþýüûúùøÿ


ÿþýüûúùøÿþ
rÑ   )rÂ   F)4Ú
__future__r   ÚsysÚ	functoolsr   Úloggingr   Úversion_infoÚtypingr   Útyping_extensionsÚImportErrorZconstantr   r   r   r   r   r   r   r   r   r   r   r   r   Úutilsr   r   r   r   r   r   r   r!   rX   r"   rZ   r4   r^   r_   rl   ry   r}   r�   r†   rŽ   rª   r­   r¸   r‹   rÑ   r   r   r   r
   Ú<module>   s\    
<$ *ÿ.,9 "\J   ÿ