o i @sUddlmZddlZddlmZddlmZddlmZm Z m Z m Z ddl m Z mZmZmZmZddlmZdd lmZmZdd lmZmZmZmZmZmZed Ze Z!e!"e#d gZ$d e%d<gZ&d e%d<e D]!Z'zee'r{e$(e'ne&(e'Wqne)ye&(e'Yqnwe$e&Z*d e%d<         d6d7d(d)Z+         d6d8d,d-Z,         d6d9d0d1Z-         d:d;d4d5Z.dS)<) annotationsN)PathLike)BinaryIO)coherence_ratioencoding_languagesmb_encoding_languagesmerge_coherence_ratios)IANA_SUPPORTEDIANA_SUPPORTED_SIMILARTOO_BIG_SEQUENCETOO_SMALL_SEQUENCETRACE) mess_ratio) CharsetMatchCharsetMatches)any_specified_encodingcut_sequence_chunks iana_nameidentify_sig_or_bomis_multi_byte_encodingshould_strip_sig_or_bomZcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)sz list[str] _mb_supported _sb_supportedIANA_SUPPORTED_MB_FIRST皙?TF皙? sequencesbytes | bytearraystepsint chunk_size thresholdfloat cp_isolationlist[str] | None cp_exclusionpreemptive_behaviourboolexplainlanguage_thresholdenable_fallbackreturnrc @ Cs t|ttfstdt||rtj} tt t t t |} | dkrDt d|r8tt t | tt|dddgdgS|durZtt d d |d d |D}ng}|durrtt d d |dd |D}ng}| ||krtt d||| d}| }|dkr| ||krt| |}t |tk} t |tk} | rtt d| n | rtt d| g}|rt|nd}|dur||tt d|t}g}g}t}t}i}d}t}d}d}d}d}d}d}t}t}t|\} }!| dur|| tt dt |!| |dd|vr#|d|tD](}"|r4|"|vr4q'|r>|"|vr>q'|"|vrEq'||"d}#| |"k}$|$oVt|"}%|"dvrh|$shtt d|"q'|"dvry|$sytt d|"q'|"|vrtt d|"q'|"|vrtt d|"q'zt|"}&Wnttfytt d|"Yq'w|r|&stt |"}'ntt!|"}'|'"|stt d|"|'|q'|r|&s||krtt d|"||q'|r|&stt d |"q'z9| r|&durt#|%dur|dtd!n |t |!td!|"d"nt#|%dur&|n|t |!d|"d"}#Wn+t$t%fy]}(zt|(t%sMtt d#|"t#|(||"WYd}(~(q'd}(~(wwt&|$sddnt |!| t| |})|&o||#duo|t |#| k}*|*rtt d$|"|#dur{|&s{t'|#}+|(|+},|,dur{|,\}-}.}/|/rCt||"|-|$|.| dus|"|ddfvr|#nd|d%}0||0||"tt d&|"t)|-d'd(d)|"|ddfvr |-d*kr |-dkrt d+|0j*|rtt t | t|0gS||0t |rA|dus||vrAd|vrAd|vrA|+}1t d+|1j*|r:tt t | t|1gSq'||"tt d,|"| ry|"dd|d-d.fvryt||"||$g|#|d%}2|"|kro|2}n |"dkrw|2}n|2}q'tt |)d/}3t,|3d0}3d}4d}5g}6g}7zMt-||"|)||$|%|!|&|# D]>}8|6|8|7t.|8||d1uodt |kod0kn|7d2|kr|4d7}4|4|3ks|$r|%durqܐqWn!t$y}(ztt d3|"t#|(|3}4d1}5WYd}(~(nd}(~(ww|5s;| r;|&s;z|td4dj/|"d5d6Wn#t$y:}(ztt d7|"t#|(||"WYd}(~(q'd}(~(ww|7rFt0|7t |7nd}9|9|ksR|4|3kr||"|"t1vrc|2t1|"|#durv|&sv|3t'|#|9gdftt d8|"|4t)|9d'd(d)| r|"dd|d-d.fvr|5st||"||$g|#|d%}2|"|kr|2}n |"dkr|2}n|2}q'tt d9|"t)|9d'd(d)|&st |"}:nt!|"}:|:rtt d:|"t#|:g};|"dkr|6D]}8t4|8||:rd;|:nd}<|;|<qt5|;}=nt5|;}=|=rtt d<|=|"t||"|9|$|=| dus&|"|ddfvr(|#nd|d%}>||>|#durE|&sE|3t'|#|9|=d1f|rT|&sT|9d=krT|d7}|"|ddfvr|9d*kr|9dkrt d+|>j*|rztt t | t|>gS||>t |r|dus||vrd|vrd|vr|+}1t d+|1j*|rtt t | t|1gS|s|&s|=rt,d>d?|=Ddd@nd}?|?dAkrd|vrd|vrd1}|2|:tt dB|"|9|?|s.|&r.|*r.|#dur.t |#| dCkr.|"dDvr.d|vr.d|vr.d1}tt dE|"|9t |#| t |#| d'|"| krOt dF|"|rFtt t | t||"gSq't |dkr|sa|sa|rgtt dG|rwt dH|j*||n2|r|dus|r|r|j6|j6ks|durt dI||n |rt dJ|||rt dK|+j*t |dnt dL|rtt t | |S)Maf Given a raw bytes sequence, return the best possibles charset usable to render str objects. If there is no results, it is a strong indicator that the source is binary/not text. By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence. And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will. The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page but never take it for granted. Can improve the performance. You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that purpose. This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32. By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain' toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging. Custom logging format and handler can be set manually. z3Expected object of type bytes or bytearray, got: {}rzszfrom_bytes..zacp_exclusion is set. use this flag for debugging purpose. limited list of encoding excluded : %s.cSr1r2r3r4r7r7r8r9~r:z^override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.rz>Trying to detect encoding from a tiny portion of ({}) byte(s).zIUsing lazy str decoding because the payload is quite large, ({}) byte(s).z@Detected declarative mark in sequence. Priority +1 given for %s.zIDetected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.ascii>utf_32utf_16z\Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.>utf_7zREncoding %s won't be tested as-is because detection is unreliable without BOM/SIG.zY%s is deemed too similar to a code page that was already considered unsuited. Continuing!zESkipping %s: already fast-tracked from a similar successful encoding.z2Encoding %s does not provide an IncrementalDecoderzbSkipping %s: definitive match already found, this encoding targets different languages (%s vs %s).zXSkipping %s: already accumulated %d same-family results after definitive match (cap=%d).zCSkipping single-byte %s: multi-byte definitive match already found.gA)encodingz9Code page %s does not fit given bytes sequence at ALL. %szpCode page %s is a multi byte encoding table and it appear that at least one character was encoded using n-bytes.)Zpreemptive_declarationzM%s fast-tracked (identical decoded payload to a prior encoding, chaos=%f %%).d)ndigitsrz.Encoding detection: %s is most likely the one.zZ%s fast-skipped (identical decoded payload to a prior encoding that failed chaos probing).r>r=TzaLazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %sgj@strict)errorsz^LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %szc%s was excluded because of initial chaos probing. Gave up %i time(s). Computed mean chaos is %f %%.z=%s passed initial chaos probing. Mean measured chaos is %f %%z&{} should target any language(s) of {},z We detected language {} using {}g{Gz?css|]\}}|VqdS)Nr7)r5_vr7r7r8 szfrom_bytes..)defaultg?zyDefinitive match found: %s (chaos=%.3f, coherence=%.2f). Encodings targeting different language families will be skipped.g\(\?> utf_16_ber/r= utf_32_ler> utf_16_leZ utf_8_sigr? utf_32_bezjMulti-byte definitive match: %s (chaos=%.3f, decoded=%d/%d=%.1f%%). Single-byte encodings will be skipped.zoEncoding detection: %s is most likely the one as we detected a BOM or SIG within the beginning of the sequence.zONothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.z7Encoding detection: %s will be used as a fallback matchz:Encoding detection: utf_8 will be used as a fallback matchz:Encoding detection: ascii will be used as a fallback matchz]Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.z=Encoding detection: Unable to determine any suitable charset.)7 isinstance bytearraybytes TypeErrorformattypeloggerlevel addHandlerexplain_handlersetLevelrlendebug removeHandlerrrlogjoinr"r r rappendsetrraddrrModuleNotFoundError ImportErrorrr intersectionstrUnicodeDecodeError LookupErrorrangehashgetroundr@bestmaxrrdecodesumr update setdefaultrr fingerprint)@rr!r#r$r&r(r)r+r,r-Zprevious_logger_levellengthZis_too_small_sequenceZis_too_large_sequenceZprioritized_encodingsZspecified_encodingZtestedZtested_but_hard_failureZtested_but_soft_failureZsoft_failure_skipZsuccess_fast_trackedZpayload_result_cacheZdefinitive_match_foundZdefinitive_target_languagesZ post_definitive_sb_success_countZPOST_DEFINITIVE_SB_CAPZmb_definitive_match_foundZfallback_asciiZ fallback_u8Zfallback_specifiedresultsZearly_stop_resultsZ sig_encodingZ sig_payloadZ encoding_ianaZdecoded_payloadZbom_or_sig_availableZstrip_sig_or_bomZis_multi_byte_decoderZ enc_languageseZr_Zmulti_byte_bonusZ payload_hashcachedZ cached_messZ cached_cdZ cached_passedZ fast_matchZprobable_resultZfallback_entryZmax_chunk_gave_upZearly_stop_countZlazy_str_hard_failureZ md_chunksZ md_ratioschunkZmean_mess_ratioZtarget_languagesZ cd_ratiosZchunk_languagesZcd_ratios_mergedZ current_matchZbest_coherencer7r7r8 from_bytes9s                                                   &                                       r{fprc Cst|||||||||| S)z Same thing than the function from_bytes but using a file pointer that is already ready. Will not close the file pointer. N)r{read) r|r!r#r$r&r(r)r+r,r-r7r7r8from_fpRsr~pathstr | bytes | PathLikec CsHt|d} t| ||||||||| WdS1swYdS)z Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode. Can raise IOError. rbN)openr~) rr!r#r$r&r(r)r+r,r-r|r7r7r8 from_pathps $rfp_or_path_or_payload!PathLike | str | BinaryIO | bytesc Cst|ttfrt|||||||||| d } | St|ttfr0t|||||||||| d } | St|||||||||| d } | S)a) Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string. Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match are disabled to be stricter around ASCII-compatible but unlikely to be a string. ) r!r#r$r&r(r)r+r,r-N)rRrhrrrTrSr{r~) rr!r#r$r&r(r)r+r,r-Zguessesr7r7r8 is_binarys\- r) rrrNNTFrT)rr r!r"r#r"r$r%r&r'r(r'r)r*r+r*r,r%r-r*r.r)r|rr!r"r#r"r$r%r&r'r(r'r)r*r+r*r,r%r-r*r.r)rrr!r"r#r"r$r%r&r'r(r'r)r*r+r*r,r%r-r*r.r) rrrNNTFrF)rrr!r"r#r"r$r%r&r'r(r'r)r*r+r*r,r%r-r*r.r*)/ __future__rloggingosrtypingrZcdrrrr Zconstantr r r r rZmdrmodelsrrutilsrrrrrr getLoggerrX StreamHandlerr[ setFormatter Formatterr__annotations__rZ_supported_encrbrfrr{r~rrr7r7r7r8s          !   !