U
    U‚gâ.  ã                   @  s.  d dl mZ d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d	d
lmZmZmZmZmZmZ e
ed�dddœdd„ƒZe
ed�dddœdd„ƒZe
ed�dddœdd„ƒZe
ed�dddœdd„ƒZe
ed�dddœdd„ƒZe
ed�dddœdd„ƒZe
ed�dddœdd„ƒZe
ed�dddœdd„ƒZe
ed�dddœd d!„ƒZ e
ed�dddœd"d#„ƒZ!e
ed�dddœd$d%„ƒZ"e
ed�dddœd&d'„ƒZ#e
ed�dddœd(d)„ƒZ$e
ed�dddœd*d+„ƒZ%e
ed�dddœd,d-„ƒZ&e
ed�dddœd.d/„ƒZ'e
e(eƒd�ddd0œd1d2„ƒZ)e
ed�dddœd3d4„ƒZ*d[d6d7dd8œd9d:„Z+e
d;d�ddd<œd=d>„ƒZ,d6d?d@œdAdB„Z-dddCœdDdE„Z.d\ddddGœdHdI„Z/dddJdKœdLdM„Z0ddddKœdNdO„Z1dPej2dQfdd7ddRdSœdTdU„Z3d]d6ddVd7ddd6dddWdXœ
dYdZ„Z4dS )^é    )ÚannotationsN)ÚIncrementalDecoder)Úaliases)Ú	lru_cache)Úfindall)Ú	Generator)ÚMultibyteIncrementalDecoderé   )ÚENCODING_MARKSÚIANA_SUPPORTED_SIMILARÚRE_POSSIBLE_ENCODING_INDICATIONÚUNICODE_RANGES_COMBINEDÚUNICODE_SECONDARY_RANGE_KEYWORDÚUTF8_MAXIMAL_ALLOCATION)ÚmaxsizeÚstrÚbool)Ú	characterÚreturnc                 C  sf   zt  | ¡}W n tk
r$   Y dS X d|kpdd|kpdd|kpdd|kpdd|kpdd|kpdd|kpdd	|kS )
NFz
WITH GRAVEz
WITH ACUTEzWITH CEDILLAzWITH DIAERESISzWITH CIRCUMFLEXz
WITH TILDEzWITH MACRONzWITH RING ABOVE©ÚunicodedataÚnameÚ
ValueError©r   Údescription© r   ú</tmp/pip-unpacked-wheel-5ringpik/charset_normalizer/utils.pyÚis_accentuated   s&    ÿþýüûúør   c                 C  s.   t  | ¡}|s| S | d¡}tt|d dƒƒS )Nú r   é   )r   ÚdecompositionÚsplitÚchrÚint)r   Z
decomposedÚcodesr   r   r   Úremove_accent,   s
    

r%   z
str | Nonec                 C  s.   t | ƒ}t ¡ D ]\}}||kr|  S qdS )zK
    Retrieve the Unicode range official name from a single character.
    N)Úordr   Úitems)r   Zcharacter_ordÚ
range_nameZ	ord_ranger   r   r   Úunicode_range7   s
    
r)   c                 C  s.   zt  | ¡}W n tk
r$   Y dS X d|kS )NFZLATINr   r   r   r   r   Úis_latinE   s
    r*   c                 C  s2   t  | ¡}d|krdS t| ƒ}|d kr*dS d|kS )NÚPTFZPunctuation©r   Úcategoryr)   ©r   Úcharacter_categoryÚcharacter_ranger   r   r   Úis_punctuationN   s    
r1   c                 C  sB   t  | ¡}d|ksd|krdS t| ƒ}|d kr2dS d|ko@|dkS )NÚSÚNTFZFormsZLor,   r.   r   r   r   Ú	is_symbol]   s    
r4   c                 C  s$   t | ƒ}|d krdS d|kp"d|kS )NFZ	EmoticonsZPictographs)r)   )r   r0   r   r   r   Úis_emoticonl   s    r5   c                 C  s.   |   ¡ s| dkrdS t | ¡}d|kp,|dkS )N>   ú<ú+õ   ï½œú>TÚZ>   ÚPcÚPoÚPd)Úisspacer   r-   )r   r/   r   r   r   Úis_separatorv   s    
r?   c                 C  s   |   ¡ |  ¡ kS ©N)ÚislowerÚisupper©r   r   r   r   Úis_case_variable€   s    rD   c                 C  s.   zt  | ¡}W n tk
r$   Y dS X d|kS )NFÚCJKr   ©r   Zcharacter_namer   r   r   Úis_cjk…   s
    rG   c                 C  s.   zt  | ¡}W n tk
r$   Y dS X d|kS )NFZHIRAGANAr   rF   r   r   r   Úis_hiragana�   s
    rH   c                 C  s.   zt  | ¡}W n tk
r$   Y dS X d|kS )NFZKATAKANAr   rF   r   r   r   Úis_katakana™   s
    rI   c                 C  s.   zt  | ¡}W n tk
r$   Y dS X d|kS )NFZHANGULr   rF   r   r   r   Ú	is_hangul£   s
    rJ   c                 C  s.   zt  | ¡}W n tk
r$   Y dS X d|kS )NFZTHAIr   rF   r   r   r   Úis_thai­   s
    rK   c                 C  s.   zt  | ¡}W n tk
r$   Y dS X d|kS )NFÚARABICr   rF   r   r   r   Ú	is_arabic·   s
    rM   c                 C  s6   zt  | ¡}W n tk
r$   Y dS X d|ko4d|kS )NFrL   zISOLATED FORMr   rF   r   r   r   Úis_arabic_isolated_formÁ   s
    rN   )r(   r   c                   s   t ‡ fdd„tD ƒƒS )Nc                 3  s   | ]}|ˆ kV  qd S r@   r   )Ú.0Úkeyword©r(   r   r   Ú	<genexpr>Í   s     z-is_unicode_range_secondary.<locals>.<genexpr>)Úanyr   rQ   r   rQ   r   Úis_unicode_range_secondaryË   s    rT   c                 C  s(   |   ¡ dko&|  ¡ dko&| dko&| dkS )NFúu   ï»¿)r>   ÚisprintablerC   r   r   r   Úis_unprintableÐ   s    
ÿþürW   é    Úbytesr#   )ÚsequenceÚsearch_zoner   c                 C  s    t | tƒst‚t| ƒ}tt| dt||ƒ… jddd�ƒ}t|ƒdkrHdS |D ]N}| ¡  	dd¡}t
 ¡ D ]0\}}||kr„|    S ||krh|    S qhqLdS )zW
    Extract using ASCII-only decoder any specified encoding in the first n-bytes.
    NÚasciiÚignore©Úerrorsr   ú-Ú_)Ú
isinstancerY   Ú	TypeErrorÚlenr   r   ÚminÚdecodeÚlowerÚreplacer   r'   )rZ   r[   Zseq_lenÚresultsZspecified_encodingÚencoding_aliasÚencoding_ianar   r   r   Úany_specified_encodingÛ   s"    
þrl   é€   )r   r   c                 C  s    | dkpt t d| › �¡jtƒS )zQ
    Verify is a specific encoding is a multi byte one based on it IANA name
    >	   Ú	utf_32_beÚutf_16Ú	utf_16_beÚutf_7Ú	utf_8_sigÚutf_8Ú	utf_32_leÚutf_32Ú	utf_16_leú
encodings.)Ú
issubclassÚ	importlibÚimport_moduler   r   )r   r   r   r   Úis_multi_byte_encodingû   s    
þör{   ztuple[str | None, bytes])rZ   r   c                 C  sJ   t D ]@}t | }t|tƒr |g}|D ]}|  |¡r$||f    S q$qdS )z9
    Identify and extract SIG/BOM in given sequence.
    )Nó    )r
   rb   rY   Ú
startswith)rZ   Úiana_encodingZmarksÚmarkr   r   r   Úidentify_sig_or_bom  s    

r€   )r~   r   c                 C  s   | dkS )N>   ro   ru   r   )r~   r   r   r   Úshould_strip_sig_or_bom"  s    r�   T)Úcp_nameÚstrictr   c                 C  sN   |   ¡  dd¡} t ¡ D ]\}}| ||fkr|  S q|rJtd| › d�ƒ‚| S )zIReturns the Python normalized encoding name (Not the IANA official name).r`   ra   zUnable to retrieve IANA for 'ú')rg   rh   r   r'   r   )r‚   rƒ   rj   rk   r   r   r   Ú	iana_name&  s    
r…   Úfloat)Úiana_name_aÚiana_name_br   c           	      C  sŒ   t | ƒst |ƒrdS t d| › �¡j}t d|› �¡j}|dd�}|dd�}d}tdƒD ]*}t|gƒ}| |¡| |¡krX|d7 }qX|d S )	Ng        rw   r]   r^   r   éÿ   r	   éþ   )r{   ry   rz   r   ÚrangerY   rf   )	r‡   rˆ   Z	decoder_aZ	decoder_bZid_aZid_bZcharacter_match_countÚiZto_be_decodedr   r   r   Úcp_similarity7  s    



r�   c                 C  s   | t ko|t |  kS )zŽ
    Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using
    the function cp_similarity.
    )r   )r‡   rˆ   r   r   r   Úis_cp_similarK  s    
þrŽ   Zcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)sÚNone)r   ÚlevelÚformat_stringr   c                 C  s:   t  | ¡}| |¡ t  ¡ }| t  |¡¡ | |¡ d S r@   )ÚloggingÚ	getLoggerÚsetLevelÚStreamHandlerÚsetFormatterÚ	FormatterÚ
addHandler)r   r�   r‘   ÚloggerÚhandlerr   r   r   Úset_logging_handlerV  s
    

r›   r‹   zGenerator[str, None, None])
Ú	sequencesrk   ÚoffsetsÚ
chunk_sizeÚbom_or_sig_availableÚstrip_sig_or_bomÚsig_payloadÚis_multi_byte_decoderÚdecoded_payloadr   c	                 c  s*  |r6|dkr6|D ]"}	||	|	| … }
|
s, q4|
V  qnð|D ]ê}	|	| }|t | ƒd krXq:| |	|	| … }|r||dkr||| }|j||rŠdndd�}
|�r|	dk�rt|dƒ}|�r|
d |… |k�rt|	|	d d	ƒD ]H}| ||… }|rø|dkrø|| }|j|dd�}
|
d |… |krÔ �qqÔ|
V  q:d S )
NFé   r]   rƒ   r^   r   r   é   éÿÿÿÿ)rd   rf   re   r‹   )rœ   rk   r�   rž   rŸ   r    r¡   r¢   r£   rŒ   ÚchunkZ	chunk_endZcut_sequenceZchunk_partial_size_chkÚjr   r   r   Úcut_sequence_chunksc  s>    

þ
ÿþr©   )rX   )T)N)5Ú
__future__r   ry   r’   r   Úcodecsr   Zencodings.aliasesr   Ú	functoolsr   Úrer   Útypingr   Z_multibytecodecr   Zconstantr
   r   r   r   r   r   r   r%   r)   r*   r1   r4   r5   r?   rD   rG   rH   rI   rJ   rK   rM   rN   rd   rT   rW   rl   r{   r€   r�   r…   r�   rŽ   ÚINFOr›   r©   r   r   r   r   Ú<module>   sx    

									
 ý ÷