U
    ùQ”i�F  ã                   @   s2  d Z ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddl	Z	ddl
mZ ddlZddlZddlZddlZddlmZ ddlZddlZddlZddlmZmZ ddlmZ dd	lmZ ddlm  mZ  ddl!Z"dd
l!m#Z#m$Z$m%Z% ddl&m'Z( ddl)m  m*Z+ ddl,m-Z-m.Z. ej/dd�dd„ ƒZ0dd„ Z1e	j2 3e	j2 4e5¡dd¡Z6e e	j2 3e6dd¡¡Z7ej/e7d�dd„ ƒZ8ej9 :ddde;dƒe<dƒe =d¡eddddgƒe<dƒ >dd¡e<dƒ >dd¡ddd… e ?d ¡j@d!d"d#�e ?d ¡j@d!d$d#�e ?d ¡j@d!d"d#�dd…ddd…f g¡d%d&„ ƒZAd'd(„ ZBd)d*„ ZCd+d,„ ZDej9 :d-ejEeCd.d/�ejEe"jFd0d/�ejEeje"jFejGd1�d2d/�ejEeje"jFd3d1�d4d/�ejEeje"jFd5d1�d6d/�g¡ej9 :d7e"jFeCg¡d8d9„ ƒƒZHd:d;„ ZId<d=„ ZJej/d>d?„ ƒZKG d@dA„ dAƒZLG dBdC„ dCƒZMej9 :dDdEgdFe#dGdHdIgƒfdJe"jN OdGdHdIgdKdd"gg¡fg¡dLdM„ ƒZPdNdO„ ZQej9 :dPdQdRdSg¡dTdU„ ƒZRe  SdV¡dWdX„ ƒZTG dYdZ„ dZejUƒZVd[d\„ ZWd]d^„ ZXd_d`„ ZYdadb„ ZZej9 :dcdddedce dfdgdg¡dhg¡didj„ ƒZ[dkdl„ Z\dmdn„ Z]e j^dodp„ ƒZ_ej9 :dqej`ejGg¡drds„ ƒZadtdu„ ZbdS )va<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
é    )ÚarrayN)Úpartial)ÚPath)Úcatch_warnings)Úget_lzma_fileÚis_platform_little_endian)Úimport_optional_dependency)Úflatten_buffer)ÚIndexÚSeriesÚperiod_range)ÚDayÚMonthEndÚmodule)Zscopec               
   C   s0   ddl m}  tƒ � | ƒ W  5 Q R £ S Q R X d S )Nr   ©Úcreate_pickle_data)Z-pandas.tests.io.generate_legacy_storage_filesr   r   r   © r   ú?/tmp/pip-unpacked-wheel-58ppya8e/pandas/tests/io/test_pickle.pyÚcurrent_pickle_data8   s    r   c                 C   s†   t |tƒrt || ¡ d S | d¡r2t | |¡ nP|dkrb|tjkrT| tjks`t‚q‚| |ks‚t‚n t	td|› d�tj
ƒ}|| |ƒ d S )NZsp_Ú	timestampZassert_Z_equal)Ú
isinstancer
   ÚtmÚassert_index_equalÚ
startswithZassert_equalÚpdZNaTÚAssertionErrorÚgetattrZassert_almost_equal)ÚresultÚexpectedÚtypZ
comparatorr   r   r   Úcompare_elementD   s    


r    ÚdataÚlegacy_pickleÚ*z*.pickle)Úparamsc                 C   s
   || j ƒS ©N)Úparam)ÚrequestÚdatapathr   r   r   r"   Y   s    s   123s   123456ÚIé   é   é   ÚB)r,   r+   é   )r,   é   ÚC)ÚorderÚFc                 C   s„   t | ƒ}t| ƒ d¡}||ks"t‚t| ttfƒr>|| ks€t‚nBt|tƒr€|jdksVt‚|jdksdt‚|j	snt‚|j
|jfks€t‚d S )NÚAr*   r-   )r	   Ú
memoryviewÚtobytesr   r   ÚbytesÚ	bytearrayÚndimÚformatÚ
contiguousÚshapeÚnbytes)r!   r   r   r   r   r   Útest_flatten_bufferc   s    

r=   c           	      C   sø  t ƒ st d¡ t | ¡}| ¡ D �]Î\}}| ¡ D �]º\}}|| | }|dk�r2|dk�r2t ||¡ |jj	|jj	ks~t
‚|jj	jrŒt
‚t |dk|dk¡ |jj	}|tdƒ tdƒksÀt
‚|tjdd� }t|tjƒsàt
‚|tjddd�ksöt
‚|tjdd	� }t|tjƒ�st
‚|tjddd
�k�sðt
‚q4|dk�r |dk�r t ||¡ t|j	tƒ�sdt
‚|j	tƒ k�svt
‚|jdk�s†t
‚t | d¡| d¡¡ q4|dk�rÂ|dk�rÂt ||¡ q4|dk�rä|dk�rät ||¡ q4t|||ƒ q4q"d S )Nz"known failure on non-little endianZseriesÚtsr   r*   r+   )Úhours)Údaysr?   )Únanoseconds)r@   rA   ÚindexZperiodÚM)Zdt_tzÚcatÚframe)Zdt_mixed_tzsZ
cat_onecolZcat_and_float)r   ÚpytestÚskipr   Úread_pickleÚitemsr   Úassert_series_equalrB   Úfreqr   Ú	normalizer   Z	Timedeltar   r   r   ZfreqstrÚshiftÚassert_frame_equalr    )	r"   r!   r   ÚdvÚdtr   r   rK   Úresr   r   r   Útest_pickles€   s<    

rR   c              	   C   s*   t |dƒ�}tj| |dd� W 5 Q R X d S ©NÚwbéÿÿÿÿ©Úprotocol©ÚopenÚpickleÚdump©ÚobjÚpathÚfhr   r   r   Úpython_pickler®   s    r`   c              
   C   s6   t | dƒ�"}| d¡ t |¡W  5 Q R £ S Q R X d S )NÚrbr   )rY   ÚseekrZ   Úload)r^   r_   r   r   r   Úpython_unpickler³   s    
rd   Úpickle_writerÚpython)ÚidZpandas_proto_defaultrV   Zpandas_proto_highestr/   Zpandas_proto_4é   Zpandas_proto_5Úwriterc                 C   sØ   | }|  ¡ D ]Æ\}}|  ¡ D ]´\}}t ¡ �ž}|||ƒ t |¡}	t|	||ƒ t|ƒ}	t|	||ƒ t|dd��}
|||ƒ |
 d¡ W 5 Q R X t|dd��}
t |
¡}	|
 d¡ W 5 Q R X t|	||ƒ W 5 Q R X qqd S )NrT   ©Úmoder   ra   )	rI   r   Úensure_cleanr   rH   r    rd   rY   rb   )r   re   ri   r!   r   rO   rP   r   r^   r   Úhandler   r   r   Útest_round_trip_current¹   s     




rn   c                  C   s(   t  ¡ } t  | jtj¡}t  | |¡ d S r%   )r   ÚmakeDataFrameÚround_trip_pathlibÚ	to_pickler   rH   rN   ©Údfr   r   r   r   Útest_pickle_path_pathlibã   s    rt   c                  C   s(   t  ¡ } t  | jtj¡}t  | |¡ d S r%   )r   ro   Zround_trip_localpathrq   r   rH   rN   rr   r   r   r   Útest_pickle_path_localpathé   s    ru   c                   C   s   dt  ¡ › d�S )NÚ__z	__.pickle)ÚuuidZuuid4r   r   r   r   Úget_random_pathô   s    rx   c                   @   sX   e Zd ZejZdd„ Zdd„ Zej	 
ddddd	g¡d
d„ ƒZdd„ Zdd„ Zdd„ ZdS )ÚTestCompressionc           	   
   C   s`  |d krt  ||¡ d S |dkr.t |d¡}nð|dkrDt |d¡}nÚ|dkr€tj|dtjd��}| 	|t
j |¡¡ W 5 Q R X nž|dkrÚt|dƒ�@}tj|dd��&}| |t
j |¡¡}| ||¡ W 5 Q R X W 5 Q R X nD|d	krðtƒ |dƒ}n.|d
k�rtdƒ |d¡}nd|› �}t|ƒ‚|dk�r\t|dƒ�$}|� | 	| ¡ ¡ W 5 Q R X W 5 Q R X d S )NÚgzipÚwÚbz2Úzip©ÚcompressionÚtarra   rj   ÚxzZzstdZ	zstandardrT   zUnrecognized compression type: )r}   r€   )ÚshutilÚcopyfilerz   rY   r|   ÚBZ2FileÚzipfileÚZipFileÚZIP_DEFLATEDÚwriteÚosr^   ÚbasenameÚtarfileÚ
gettarinfoÚaddfiler   r   Ú
ValueErrorÚread)	ÚselfZsrc_pathÚ	dest_pathr   Úfr_   r€   ÚtarinfoÚmsgr   r   r   Úcompress_fileü   s2     "


zTestCompression.compress_filec                 C   s²   |}|d }|d }t  |¡�Š}t  |¡�t}t  ¡ }|j||d� t j||d��*}	t|dƒ�}
|
 |	 ¡ ¡ W 5 Q R X W 5 Q R X tj	|d d�}t  
||¡ W 5 Q R X W 5 Q R X d S )Nú.compressedú.rawr~   rT   )r   rl   ro   rq   Údecompress_filerY   rˆ   r�   r   rH   rN   )r�   r   rx   ÚbaseÚpath1Úpath2Úp1Úp2rs   r’   r_   Údf2r   r   r   Útest_write_explicit  s    "z#TestCompression.test_write_explicitr   Ú ÚNoneÚbadZ7zc              
   C   sJ   t jtdd��2 t |¡�}t ¡ }|j||d� W 5 Q R X W 5 Q R X d S )NzUnrecognized compression type)Úmatchr~   )rF   ZraisesrŽ   r   rl   ro   rq   )r�   r   rx   r^   rs   r   r   r   Útest_write_explicit_bad/  s    z'TestCompression.test_write_explicit_badc                 C   s¾   |}|| }|d }| j  | ¡ ¡}t |¡�†}t |¡�p}t ¡ }	|	 |¡ tj||d��*}
t|dƒ�}| 	|
 
¡ ¡ W 5 Q R X W 5 Q R X tj|d d�}t |	|¡ W 5 Q R X W 5 Q R X d S )Nr—   r~   rT   )Ú_extension_to_compressionÚgetÚlowerr   rl   ro   rq   r˜   rY   rˆ   r�   r   rH   rN   )r�   Úcompression_extrx   r™   rš   r›   r   rœ   r�   rs   r’   r_   rž   r   r   r   Útest_write_infer6  s    
"z TestCompression.test_write_inferc           
   
   C   s„   |}|d }|d }t  |¡�\}t  |¡�F}t  ¡ }|j|d d� | j|||d� tj||d�}	t  ||	¡ W 5 Q R X W 5 Q R X d S )Nr—   r–   r~   )r   rl   ro   rq   r•   r   rH   rN   )
r�   r   rx   r™   rš   r›   rœ   r�   rs   rž   r   r   r   Útest_read_explicitL  s    z"TestCompression.test_read_explicitc              
   C   s�   |}|d }|| }| j  | ¡ ¡}t |¡�X}t |¡�B}t ¡ }	|	j|d d� | j|||d� t 	|¡}
t 
|	|
¡ W 5 Q R X W 5 Q R X d S )Nr—   r~   )r¥   r¦   r§   r   rl   ro   rq   r•   r   rH   rN   )r�   r¨   rx   r™   rš   r›   r   rœ   r�   rs   rž   r   r   r   Útest_read_infer^  s    
zTestCompression.test_read_inferN)Ú__name__Ú
__module__Ú__qualname__ÚicomZextension_to_compressionr¥   r•   rŸ   rF   ÚmarkÚparametrizer¤   r©   rª   r«   r   r   r   r   ry   ù   s   
ry   c                   @   s*   e Zd Zej dddddg¡dd„ ƒZdS )	ÚTestProtocolrW   rU   r   r*   r+   c              	   C   sF   t  |¡�2}t  ¡ }|j||d� t |¡}t  ||¡ W 5 Q R X d S )NrV   ©r   rl   ro   rq   r   rH   rN   )r�   rW   rx   r^   rs   rž   r   r   r   Ú	test_readx  s
    
zTestProtocol.test_readN)r¬   r­   r®   rF   r°   r±   r´   r   r   r   r   r²   w  s   r²   Úpickle_fileÚexcolsztest_py27.pklÚaÚbÚcztest_mi_py27.pklr3   c                 C   s*   | ddd|ƒ}t  |¡}t |j|¡ d S )NÚior!   rZ   )r   rH   r   r   Úcolumns)r(   rµ   r¶   r^   rs   r   r   r   Útest_unicode_decode_error�  s    
r¼   c               
   C   sl   t  ¡ �Z} t  ¡ }t| dƒ�}| |¡ W 5 Q R X t| dƒ�}t |¡}W 5 Q R X t  ||¡ W 5 Q R X d S )NrT   ra   )r   rl   ro   rY   rq   r   rH   rN   )r^   rs   r_   r   r   r   r   Útest_pickle_buffer_roundtripš  s    
r½   Úmockurlzhttp://url.comzftp://test.comzhttp://gzip.comc              	      sp   dd„ }G dd„ dƒ‰ t  ¡ �H‰‡ ‡fdd„}t  ¡ }||ˆƒ |  d|¡ t |¡}t  ||¡ W 5 Q R X d S )Nc              	   S   s*   t |dƒ�}tj| |dd� W 5 Q R X d S rS   rX   r\   r   r   r   r`   ­  s    z3test_pickle_generalurl_read.<locals>.python_picklerc                   @   s:   e Zd Zddœdd„Zdd„ Zdd„ Zd	d
„ Zdd„ ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponseN©Úreturnc                 S   s.   t |dƒ| _d|kr ddi| _n
ddi| _d S )Nra   rz   zContent-Encodingr    )rY   ÚfileÚheaders)r�   r^   r   r   r   Ú__init__²  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S   s   | S r%   r   ©r�   r   r   r   Ú	__enter__¹  s    z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W   s   |   ¡  d S r%   )Úclose)r�   Úargsr   r   r   Ú__exit__¼  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S   s
   | j  ¡ S r%   )rÁ   r�   rÄ   r   r   r   r�   ¿  s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S   s
   | j  ¡ S r%   )rÁ   rÆ   rÄ   r   r   r   rÆ   Â  s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.close)r¬   r­   r®   rÃ   rÅ   rÈ   r�   rÆ   r   r   r   r   ÚMockReadResponse±  s
   rÉ   c                     s   ˆ ˆƒS r%   r   )rÇ   Úkwargs©rÉ   r^   r   r   Úmock_urlopen_readÇ  s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   rl   ro   Úsetattrr   rH   rN   )Zmonkeypatchr¾   r`   rÌ   rs   r   r   rË   r   Útest_pickle_generalurl_read©  s    


rÎ   Zfsspecc               	   C   sD   t  ¡ �2 d} t  ¡ }| | ¡ t | ¡}t  ||¡ W 5 Q R X d S )Nzmemory://afiler³   )r¾   rs   r   r   r   r   Útest_pickle_fsspec_roundtripÑ  s    


rÏ   c                   @   s   e Zd Zddœdd„ZdS )ÚMyTzNr¿   c                 C   s   d S r%   r   rÄ   r   r   r   rÃ   Ü  s    zMyTz.__init__)r¬   r­   r®   rÃ   r   r   r   r   rÐ   Û  s   rÐ   c                  C   sD   t td�tƒ f} t | ¡}t |d | d ¡ t|d tƒs@t‚d S )N)Zdtyper   r*   )r   ÚobjectrÐ   r   Úround_trip_picklerJ   r   r   )r   r   r   r   r   Útest_read_pickle_with_subclassà  s    
rÓ   c              	   C   s–   t  ¡ }t  ¡ � }|j|| d� t|ƒ ¡ }W 5 Q R X t ¡ }|j|| d� | d¡ | 	¡ |ksn| dksnt
‚tj|| d�}| d¡ t  ||¡ dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    r~   r   )rz   r}   r€   N)r   ro   rl   rq   r   Ú
read_bytesrº   ÚBytesIOrb   Úgetvaluer   r   rH   rN   )r   rs   r^   Ú	referenceÚbufferZread_dfr   r   r   Ú%test_pickle_binary_object_compressioné  s    


rÙ   c                 C   s8   | }|}dd„ }||ƒ ||j ƒ ||ƒ ||j ƒ d S )Nc                 S   s   t  | ¡}t  | |¡ d S r%   )r   rÒ   rN   )rE   Ú	unpickledr   r   r   Ú_test_roundtrip  s    
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)ÚT)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdrE   rÛ   r   r   r   Ú+test_pickle_dataframe_with_multilevel_index  s    
rÝ   c                  C   sB   t dddd�} ttj t| ƒ¡| ƒ}t |¡}|jj	dks>t
‚d S )Nz1/1/2011z1/1/2012rC   )rK   )r   r   ÚnpÚrandomZrandnÚlenr   rÒ   rB   rK   r   )Zprngr>   Znew_tsr   r   r   Ú"test_pickle_timeseries_periodindex  s    
rá   Únamei	  g     Hˆ@iÑ  é   )r*   r+   c                 C   s$   t  t j| d�¡}|j| ks t‚d S )N)râ   )r   rÒ   ZmakeTimeSeriesrâ   r   )râ   rÚ   r   r   r   Útest_pickle_preserve_name  s    rä   c                 C   s   t  | ¡}t  || ¡ d S r%   ©r   rÒ   rJ   )Zdatetime_seriesZunp_tsr   r   r   Útest_pickle_datetimes%  s    
ræ   c                 C   s   t  | ¡}t  || ¡ d S r%   rå   )Zstring_seriesZ
unp_seriesr   r   r   Útest_pickle_strings*  s    
rç   c                  C   sf   t tdƒƒ d¡jdg } t | ¡}|jjd jdks:t	‚|jjd j
dksPt	‚t |dg | ¡ d S )NÚabcÚcategoryr   r*   )r*   T)r   ÚlistZastypeZilocr   rÒ   Z_mgrÚblocksr8   r   r;   rJ   )ZserrQ   r   r   r   Ú test_pickle_preserves_block_ndim/  s
    
rì   rW   c                 C   s@   t  tdƒ¡}t t|j| |d�tt j|d�¡}t ||¡ d S )Ni † )rW   r   r~   )	r   Ú	DataFrameÚranger   rp   r   rq   rH   rN   )rW   r   rs   r   r   r   r   Ú%test_pickle_big_dataframe_compression<  s    þrï   c               	   C   sN   t j tdd¡} t| dƒ�}t |¡}W 5 Q R X tjg g d�}t	 
||¡ d S )Nz1.2.4zempty_frame_v1_2_4-GH#42345.pklra   )rB   r»   )r‰   r^   ÚjoinÚlegacy_dirnamerY   rZ   rc   r   rí   r   rN   )r^   Úfdrs   r   r   r   r   Ú#test_pickle_frame_v124_unpickle_130G  s
    ró   )cÚ__doc__r   r|   ÚdatetimeÚ	functoolsr   Úglobrz   rº   r‰   Úpathlibr   rZ   r‚   r‹   rw   Úwarningsr   r…   ZnumpyrÞ   rF   Zpandas.compatr   r   Zpandas.compat._optionalr   Zpandas.compat.compressorsr	   Zpandas.util._test_decoratorsÚutilZ_test_decoratorsÚtdZpandasr   r
   r   r   Zpandas._testingZ_testingr   Zpandas.io.commonÚcommonr¯   Zpandas.tseries.offsetsr   r   Zfixturer   r    r^   rð   ÚdirnameÚ__file__rñ   Úfilesr"   r°   r±   r7   r4   ÚPickleBufferÚcastZarangeZreshaper=   rR   r`   rd   r&   rq   ÚHIGHEST_PROTOCOLrn   rt   ru   rx   ry   r²   Z
MultiIndexZfrom_arraysr¼   r½   rÎ   Z
skip_if_norÏ   ÚtzinforÐ   rÓ   rÙ   rÝ   rá   rä   ræ   rç   Zskip_array_manager_invalid_testrì   ÚDEFAULT_PROTOCOLrï   ró   r   r   r   r   Ú<module>   sÜ   



	$õþ
.þþøþ
~
þþþ

 ÿ
%
		 ÿ



