U }lj4 @shUddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZddlmZd d lmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"d d d ddZ#d dd ddZ$e edd d d ddZ%e&dde'DZ(de)d<dde(DZ*de)d<d dd dd Z+d dd d!d"Z,d dd d#d$Z-d dd d%d&Z.d dd d'd(Z/d dd d)d*Z0e edd dd d+d,Z1e edd dd d-d.Z2e edd dd d/d0Z3e edd dd d1d2Z4e edd dd d3d4Z5e edd dd d5d6Z6e edd dd d7d8Z7e edd dd d9d:Z8e edd dd d;d<Z9d dd=d>d?Z:e edd dd d@dAZ;didCd ddDdEdFZd ddOdPdQZ?djd dd dSdTdUZ@d d dVdWdXdYZAd d ddWdZd[ZBd\ejCd]fd d d d^d_d`daZDdkdCd dcd ddddddddedf dgdhZEdS)l) annotationsN) bisect_right)IncrementalDecoder)aliases) lru_cache)findall) Generator)ENCODING_MARKSIANA_SUPPORTED_SIMILARRE_POSSIBLE_ENCODING_INDICATIONUNICODE_RANGES_COMBINED_SECONDARY_RANGE_NAMESUTF8_MAXIMAL_ALLOCATIONCOMMON_CJK_CHARACTERS_LATIN_CJK_HANGUL _KATAKANA _HIRAGANA_THAI_ARABIC_ARABIC_ISOLATED_FORM_ACCENT_KEYWORDS _ACCENTUATEDstrint) characterreturncCszt|}Wntk r$YdSXd}d|kr:|tO}d|krJ|tO}d|krZ|tO}d|krj|tO}d|krz|tO}d|kr|tO}d|kr|t O}d |kr|t O}t D]}||kr|t O}qq|S) zRCompute all name-based classification flags with a single unicodedata.name() call.rZLATINCJKZHANGULZKATAKANAZHIRAGANAZTHAIZARABICz ISOLATED FORM) unicodedataname ValueErrorrrrrrrrrrr)rdescflagskwr&Usr8zlist[tuple[int, int, str]]_UNICODE_RANGES_SORTEDcCsg|] }|dqS)rr&)r7er&r&r' Ysr;z list[int]_UNICODE_RANGE_STARTSz str | NonecCs<t|}tt|d}|dkr8t|\}}}||kr8|SdS)zK Retrieve the Unicode range official name from a single character. r rN)ordrr<r9)rZ character_ordidxr5r6r!r&r&r' unicode_range\sr?cCstt|t@Sr*)r)r(rr+r&r&r'is_latinlsr@cCs2t|}d|krdSt|}|dkr*dSd|kS)NPTFZ Punctuationr categoryr?rcharacter_categorycharacter_ranger&r&r'is_punctuationps rGcCsBt|}d|ksd|krdSt|}|dkr2dSd|ko@|dkS)NSNTFZFormsZLorBrDr&r&r' is_symbol~s rJcCs$t|}|dkrdSd|kp"d|kS)NFZ EmoticonsZ Pictographs)r?)rrFr&r&r' is_emoticonsrKcCs.|s|dkrdSt|}d|kp,|dkS)N>+｜>PdPcPo)isspacer rC)rrEr&r&r' is_separators rUcCs||kSr*)islowerisupperr+r&r&r'is_case_variablesrXcCstt|t@Sr*)r)r(rr+r&r&r'is_cjksrYcCstt|t@Sr*)r)r(rr+r&r&r' is_hiraganasrZcCstt|t@Sr*)r)r(rr+r&r&r' is_katakanasr[cCstt|t@Sr*)r)r(rr+r&r&r' is_hangulsr\cCstt|t@Sr*)r)r(rr+r&r&r'is_thaisr]cCstt|t@Sr*)r)r(rr+r&r&r' is_arabicsr^cCstt|t@Sr*)r)r(rr+r&r&r'is_arabic_isolated_formsr_cCs|tkSr*)rr+r&r&r'is_cjk_uncommonsr`) range_namercCs|tkSr*)r)rar&r&r'is_unicode_range_secondarysrbcCs$| o"| o"|dko"|dkS)Nu)rT isprintabler+r&r&r'is_unprintables re zbytes | bytearray)sequence search_zonerc Cst|ttfstt|}|dt||jddd}|}d|krRd|krRdStt |}t|dkrldS|D]N}| dd }t D]0\}}||kr|S||kr|SqqpdS) zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. NasciiignoreerrorsZcodingcharsetr-_) isinstancebytes bytearray TypeErrorlenmindecodelowerrr replaceritems) rgrhZseq_lenZ decoded_zoneZ lowered_zoneresultsZspecified_encodingencoding_alias encoding_ianar&r&r'any_specified_encodings0  r})r!rc CsR|dkr dSdD]<}zt||Wntttfk rFYqYnXdSdS)zQ Verify is a specific encoding is a multi byte one based on it IANA name > utf_16_be utf_32_leutf_32 utf_16_le utf_8_sig utf_32_beutf_8utf_16utf_7T)Z _codecs_cnZ _codecs_hkZ_codecs_iso2022Z _codecs_jpZ _codecs_krZ _codecs_twF) importlib import_moduleZgetcodec ImportErrorAttributeError LookupError)r!Zproviderr&r&r'is_multi_byte_encodings   rztuple[str | None, bytes])rgrcCsJtD]@}t|}t|tr |g}|D]}||r$||fSq$qdS)z9 Identify and extract SIG/BOM in given sequence. )N)r rprq startswith)rg iana_encodingZmarksmarkr&r&r'identify_sig_or_bom/s  r)rrcCs|dkS)N>rrr&)rr&r&r'should_strip_sig_or_bomAsrT)cp_namestrictrcCsN|dd}tD]\}}|||fkr|Sq|rJtd|d|S)zIReturns the Python normalized encoding name (Not the IANA official name).rnrozUnable to retrieve IANA for '')rwrxrryr")rrr{r|r&r&r' iana_nameEs  rfloat) iana_name_a iana_name_brc Cst|st|rdStd|j}td|j}|dd}|dd}d}tdD]*}t|g}||||krX|d7}qX|dS)Ngz encodings.rjrkrr )rrrrrangerqrv) rrZ decoder_aZ decoder_bZid_aZid_bZcharacter_match_countiZ to_be_decodedr&r&r' cp_similarityVs     rcCs|tko|t|kS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r )rrr&r&r' is_cp_similarjs rZcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)sNone)r!level format_stringrcCs:t|}||t}|t|||dSr*)logging getLoggersetLevel StreamHandler setFormatter Formatter addHandler)r!rrloggerhandlerr&r&r'set_logging_handlerus   rFrrqzGenerator[str, None, None]) sequencesr|offsets chunk_sizebom_or_sig_availablestrip_sig_or_bom sig_payloadis_multi_byte_decoderdecoded_payloaddeferred_decodingrc cst|r4|s4|D]"} || | |} | s(q0| Vq n<| r|s@|n|t|d} |D](} | | | |} | spq~t| |VqTn|D]} | |}|t|dkrq|| | |} |r|s|| } | j||rdndd} |rh| dkrht|d}|rh| d||krht| | ddD]L}|||} |r>|s>|| } | j|dd} | d||krqhq| VqdS) Nrjrrkrr/)rtrrvrur)rr|rrrrrrrrrchunkZ base_bytesZ cut_sequenceZ chunk_endZchunk_partial_size_chkjr&r&r'cut_sequence_chunkssN       r)rf)T)NF)F __future__rrrr bisectrcodecsrZencodings.aliasesr functoolsrrertypingrZconstantr r r r rrrrrrrrrrrrrr(r,r4sortedryr9__annotations__r<r?r@rGrJrKrUrXrYrZr[r\r]r^r_r`rbrer}rrrrrrINFOrrr&r&r&r's|       L"    +)