U
    ùQ”iüi  ã                   @  s(  d Z ddlmZ ddlmZ ddlmZmZ ddlZddlm	Z	 ddl
ZddlmZmZmZ ddlmZmZ ddlZdd	lmZmZ dd
lmZ ddlmZmZmZmZmZ ddlm Z m!Z! ddl"m#  m$  m%Z& ddl'm(Z( dddœdd„Z)ddddœdd„Z*G dd„ dƒZ+G dd„ de(ej,ƒZ-dS )a�  
Read SAS7BDAT files

Based on code written by Jared Hobbs:
  https://bitbucket.org/jaredhobbs/sas7bdat

See also:
  https://github.com/BioStatMatt/sas7bdat

Partial documentation of the file format:
  https://cran.r-project.org/package=sas7bdat/vignettes/sas7bdat.pdf

Reference for binary data compression:
  http://collaboration.cmc.ec.gc.ca/science/rpn/biblio/ddj/Website/articles/CUJ/1992/9210/ross/ross.htm
é    )Úannotations)Úabc)ÚdatetimeÚ	timedeltaN)Úcast)ÚCompressionOptionsÚFilePathÚ
ReadBuffer)ÚEmptyDataErrorÚOutOfBoundsDatetime)Ú	DataFrameÚisna)Ú
get_handle)Úread_double_with_byteswapÚread_float_with_byteswapÚread_uint16_with_byteswapÚread_uint32_with_byteswapÚread_uint64_with_byteswap)ÚParserÚget_subheader_index)Ú
ReaderBaseÚfloatÚstr©Zsas_datetimeÚunitc                 C  sV   t | ƒrtjS |dkr,tdddƒt| d� S |dkrJtdddƒt| d� S tdƒ‚d S )NÚsé¨  é   )ÚsecondsÚd)Údayszunit must be 'd' or 's')r   ÚpdZNaTr   r   Ú
ValueErrorr   © r#   ú:/tmp/pip-unpacked-wheel-58ppya8e/pandas/io/sas/sas7bdat.pyÚ_parse_datetime<   s    r%   z	pd.Series)Úsas_datetimesr   Úreturnc                 C  sJ   zt j| |dd�W S  tk
rD   | jt|d�}tt j|ƒ}| Y S X dS )aÉ  
    Convert to Timestamp if possible, otherwise to datetime.datetime.
    SAS float64 lacks precision for more than ms resolution so the fit
    to datetime.datetime is ok.

    Parameters
    ----------
    sas_datetimes : {Series, Sequence[float]}
       Dates or datetimes in SAS
    unit : {str}
       "d" if the floats represent dates, "s" for datetimes

    Returns
    -------
    Series
       Series of datetime64 dtype or datetime.datetime.
    z
1960-01-01)r   Úorigin©r   N)r!   Zto_datetimer   Úapplyr%   r   ÚSeries)r&   r   Zs_seriesr#   r#   r$   Ú_convert_datetimesJ   s    r,   c                   @  sX   e Zd ZU ded< ded< ded< ded< ded< ded	< ddddddd
dœdd„ZdS )Ú_ColumnÚintÚcol_idústr | bytesÚnameÚlabelÚformatÚbytesÚctypeÚlengthÚNone)r/   r1   r2   r3   r5   r6   r'   c                 C  s(   || _ || _|| _|| _|| _|| _d S ©N)r/   r1   r2   r3   r5   r6   )Úselfr/   r1   r2   r3   r5   r6   r#   r#   r$   Ú__init__l   s    
z_Column.__init__N)Ú__name__Ú
__module__Ú__qualname__Ú__annotations__r:   r#   r#   r#   r$   r-   d   s   
r-   c                   @  sâ  e Zd ZU dZded< ded< dSd	d
d
ddd
d
dddœ	dd„Zddœdd„Zddœdd„Zddœdd„Zddœdd„Z	ddœdd„Z
ddœdd „Zddd!œd"d#„Zdddd$œd%d&„Zddd'œd(d)„Zddd*d+œd,d-„Zddœd.d/„Zd
dœd0d1„Zddœd2d3„Zddœd4d5„Zdddd+œd6d7„Zdddd+œd8d9„Zdddd+œd:d;„Zdddd+œd<d=„Zdddd+œd>d?„Zdddd+œd@dA„Zdddd+œdBdC„Zdddd+œdDdE„ZdTdddFœdGdH„ZdIdJ„ ZddœdKdL„ZdMdN„ ZdOd*dPœdQdR„Z dS )UÚSAS7BDATReaderaË  
    Read SAS files in SAS7BDAT format.

    Parameters
    ----------
    path_or_buf : path name or buffer
        Name of SAS file or file-like object pointing to SAS file
        contents.
    index : column identifier, defaults to None
        Column to use as index.
    convert_dates : bool, defaults to True
        Attempt to convert dates to Pandas datetime values.  Note that
        some rarely used SAS date formats may be unsupported.
    blank_missing : bool, defaults to True
        Convert empty strings to missing values (SAS uses blanks to
        indicate missing character variables).
    chunksize : int, defaults to None
        Return SAS7BDATReader object for iterations, returns chunks
        with given number of lines.
    encoding : str, 'infer', defaults to None
        String encoding acc. to Python standard encodings,
        encoding='infer' tries to detect the encoding from the file header,
        encoding=None will leave the data in binary format.
    convert_text : bool, defaults to True
        If False, text variables are left as raw bytes.
    convert_header_text : bool, defaults to True
        If False, header text, including column names, are left as raw
        bytes.
    r.   Ú_int_lengthzbytes | NoneÚ_cached_pageNTÚinferzFilePath | ReadBuffer[bytes]Úboolz
int | Nonez
str | Noner   r7   )	Úpath_or_bufÚconvert_datesÚblank_missingÚ	chunksizeÚencodingÚconvert_textÚconvert_header_textÚcompressionr'   c
           
   	   C  sú   || _ || _|| _|| _|| _|| _|| _d| _d| _g | _	g | _
g | _g | _g | _d | _g | _g | _g | _d| _d| _d| _t|dd|	d�| _| jj| _| j| j| j| j| j| j| j| jd g	| _ z|  !¡  |  "¡  W n t#k
rô   |  $¡  ‚ Y nX d S )Nzlatin-1ó    r   ÚrbF)Zis_textrK   )%ÚindexrE   rF   rG   rH   rI   rJ   Údefault_encodingrK   Úcolumn_names_rawÚcolumn_namesÚcolumn_formatsÚcolumnsÚ%_current_page_data_subheader_pointersrA   Ú_column_data_lengthsÚ_column_data_offsetsÚ_column_typesÚ_current_row_in_file_indexZ_current_row_on_page_indexr   ÚhandlesÚhandleÚ_path_or_bufÚ_process_rowsize_subheaderÚ_process_columnsize_subheaderÚ_process_subheader_countsÚ_process_columntext_subheaderÚ_process_columnname_subheaderÚ#_process_columnattributes_subheaderÚ_process_format_subheaderÚ_process_columnlist_subheaderÚ_subheader_processorsÚ_get_propertiesÚ_parse_metadataÚ	ExceptionÚclose)
r9   rD   rN   rE   rF   rG   rH   rI   rJ   rK   r#   r#   r$   r:   ¡   sX       ÿ
÷zSAS7BDATReader.__init__z
np.ndarray)r'   c                 C  s   t j| jt jd�S )z5Return a numpy int64 array of the column data lengths©Údtype)ÚnpÚasarrayrU   Úint64©r9   r#   r#   r$   Úcolumn_data_lengthsà   s    z"SAS7BDATReader.column_data_lengthsc                 C  s   t j| jt jd�S )z0Return a numpy int64 array of the column offsetsri   )rk   rl   rV   rm   rn   r#   r#   r$   Úcolumn_data_offsetsä   s    z"SAS7BDATReader.column_data_offsetsc                 C  s   t j| jt  d¡d�S )zj
        Returns a numpy character array of the column types:
           s (string) or d (double)
        ZS1ri   )rk   rl   rW   rj   rn   r#   r#   r$   Úcolumn_typesè   s    zSAS7BDATReader.column_typesc                 C  s   | j  ¡  d S r8   )rY   rh   rn   r#   r#   r$   rh   ï   s    zSAS7BDATReader.closec                 C  s
  | j  d¡ | j  d¡| _| jdttjƒ… tjkr<tdƒ‚|  tj	tj
¡}|tjkrtd| _d| _tj| _tj| _nd| _tj| _tj| _d| _|  tjtj¡}|tjkr²tj}nd}|  tjtj¡}|dkrâd	| _tjd
k| _nd| _tjdk| _|  tjtj¡d }|tj k�r6tj | | _!| j"dk�rD| j!| _"nd|› d�| _!t#dddƒ}|  $tj%| tj&¡}|t'j(|dd� | _)|  $tj*| tj+¡}|t'j(|dd� | _,|  -tj.| tj/¡| _0| j  | j0d ¡}|  j|7  _t| jƒ| j0k�rðtdƒ‚|  -tj1| tj2¡| _3d S )Nr   i   z'magic number mismatch (not a SAS file?)Té   Fé   ó   ú<Úbigú>ÚlittlerB   zunknown (code=ú)r   r   r   r)   z*The SAS7BDAT file appears to be truncated.)4r[   ÚseekÚreadrA   ÚlenÚconstÚmagicr"   Ú_read_bytesZalign_1_offsetZalign_1_lengthZu64_byte_checker_valueÚU64r@   Zpage_bit_offset_x64Ú_page_bit_offsetZsubheader_pointer_length_x64Ú_subheader_pointer_lengthZpage_bit_offset_x86Zsubheader_pointer_length_x86Zalign_2_offsetZalign_2_lengthZalign_1_checker_valueZalign_2_valueZendianness_offsetZendianness_lengthÚ
byte_orderÚsysÚ	byteorderÚneed_byteswapZencoding_offsetZencoding_lengthZencoding_namesZinferred_encodingrH   r   Ú_read_floatZdate_created_offsetZdate_created_lengthr!   Zto_timedeltaZdate_createdZdate_modified_offsetZdate_modified_lengthZdate_modifiedÚ
_read_uintZheader_size_offsetZheader_size_lengthÚheader_lengthZpage_size_offsetZpage_size_lengthÚ_page_length)r9   ÚbufZalign1ÚepochÚxr#   r#   r$   re   ò   sj    



 ÿ ÿ ÿ ÿzSAS7BDATReader._get_propertiesr   c                 C  s(   | j | jpdd�}|jr$|  ¡  t‚|S )Nr   )Únrows)r{   rG   Úemptyrh   ÚStopIteration)r9   Údar#   r#   r$   Ú__next__8  s
    zSAS7BDATReader.__next__)ÚoffsetÚwidthc                 C  sR   | j d k	st‚|dkr&t| j || jƒS |dkr>t| j || jƒS |  ¡  tdƒ‚d S )Nrs   rr   zinvalid float width)rA   ÚAssertionErrorr   r†   r   rh   r"   ©r9   r“   r”   r#   r#   r$   r‡   @  s      ÿ  ÿzSAS7BDATReader._read_float)r“   r”   r'   c                 C  s‚   | j d k	st‚|dkr&|  |d¡d S |dkr>t| j || jƒS |dkrVt| j || jƒS |dkrnt| j || jƒS |  ¡  tdƒ‚d S )Nr   r   é   rs   rr   zinvalid int width)	rA   r•   r   r   r†   r   r   rh   r"   r–   r#   r#   r$   rˆ   O  s.      ÿ  ÿ  ÿzSAS7BDATReader._read_uint)r“   r6   c                 C  sB   | j d k	st‚|| t| j ƒkr0|  ¡  tdƒ‚| j ||| … S )NzThe cached page is too small.)rA   r•   r|   rh   r"   ©r9   r“   r6   r#   r#   r$   r   c  s
    zSAS7BDATReader._read_bytesr0   )r“   r6   r'   c                 C  s   |   |  ||¡ d¡¡S )Nó     )Ú_convert_header_textr   Úrstripr˜   r#   r#   r$   Ú_read_and_convert_header_textj  s    ÿz,SAS7BDATReader._read_and_convert_header_textc                 C  sN   d}|sJ| j  | j¡| _t| jƒdkr(qJt| jƒ| jkr@tdƒ‚|  ¡ }qd S )NFr   z2Failed to read a meta data page from the SAS file.)r[   r{   rŠ   rA   r|   r"   Ú_process_page_meta)r9   Údoner#   r#   r$   rf   o  s    zSAS7BDATReader._parse_metadatac                 C  sZ   |   ¡  tjtjtjg }| j|kr,|  ¡  | jtjk}| jtjk}t|pV|pV| j	g kƒS r8   )
Ú_read_page_headerr}   Úpage_meta_typesZpage_amd_typeÚpage_mix_typeÚ_current_page_typeÚ_process_page_metadataÚpage_data_typerC   rT   )r9   ÚptZis_data_pageZis_mix_pager#   r#   r$   r�   y  s    
ÿýz!SAS7BDATReader._process_page_metac                 C  s^   | j }tj| }|  |tj¡tj@ | _tj| }|  |tj¡| _	tj
| }|  |tj¡| _d S r8   )r�   r}   Zpage_type_offsetrˆ   Zpage_type_lengthZpage_type_mask2r¢   Zblock_count_offsetZblock_count_lengthZ_current_page_block_countZsubheader_count_offsetZsubheader_count_lengthÚ_current_page_subheaders_count)r9   Ú
bit_offsetZtxr#   r#   r$   rŸ   †  s    
ÿ

 ÿz SAS7BDATReader._read_page_headerc                 C  s  | j }t| jƒD �]}tj| }|| j|  }|  || j¡}|| j7 }|  || j¡}|| j7 }|  |d¡}|d7 }|  |d¡}|dks|tjkr’q|  	|| j¡}	t
|	ƒ}
| j|
 }|d k�r|tjdfk}|tjk}| jrô|rô|rô| j ||f¡ n|  ¡  td|	› �ƒ‚q|||ƒ qd S )Nr   r   zUnknown subheader signature )r�   Úranger¦   r}   Zsubheader_pointers_offsetr‚   rˆ   r@   Ztruncated_subheader_idr   r   rd   Zcompressed_subheader_idZcompressed_subheader_typerK   rT   Úappendrh   r"   )r9   r§   Úir“   Ztotal_offsetZsubheader_offsetZsubheader_lengthZsubheader_compressionZsubheader_typeZsubheader_signatureZsubheader_indexZsubheader_processorÚf1Úf2r#   r#   r$   r£   “  s>    


ÿþ


ÿÿz%SAS7BDATReader._process_page_metadatac                 C  sÒ   | j }|}|}| jr&|d7 }|d7 }n|d7 }|d7 }|  |tj|  |¡| _|  |tj|  |¡| _|  |tj|  |¡| _	|  |tj
|  |¡| _tj| }|  || |¡| _|  |d¡| _|  |d¡| _d S )Niª  iÂ  ib  iz  r—   )r@   r€   rˆ   r}   Zrow_length_offset_multiplierZ
row_lengthZrow_count_offset_multiplierÚ	row_countZcol_count_p1_multiplierÚcol_count_p1Zcol_count_p2_multiplierÚcol_count_p2Z'row_count_on_mix_page_offset_multiplierZ_mix_page_row_countÚ_lcsÚ_lcp)r9   r“   r6   Úint_lenZ
lcs_offsetZ
lcp_offsetZmxr#   r#   r$   r\   ¾  s8    
þþ ÿ ÿ
z)SAS7BDATReader._process_rowsize_subheaderc                 C  sT   | j }||7 }|  ||¡| _| j| j | jkrPtd| j› d| j› d| j› d�ƒ d S )Nz Warning: column count mismatch (z + z != z)
)r@   rˆ   Úcolumn_countr®   r¯   Úprint)r9   r“   r6   r²   r#   r#   r$   r]   Ü  s    ÿz,SAS7BDATReader._process_columnsize_subheaderc                 C  s   d S r8   r#   r˜   r#   r#   r$   r^   ç  s    z(SAS7BDATReader._process_subheader_countsc           	      C  sš  || j 7 }|  |tj¡}|  ||¡}|d|…  d¡}| j |¡ t| jƒdk�r–d}tj	D ]}||kr\|}q\|| _
|| j 8 }|d }| jr”|d7 }|  || j¡}| d¡}|dkrðd| _|d }| jrÐ|d7 }|  || j¡}|d| j… | _nŒ|tjk�r4|d	 }| j�r|d7 }|  || j¡}|d| j… | _nH| jdk�r|d| _|d }| j�r^|d7 }|  || j¡}|d| j… | _t| d
ƒ�r–|  | j¡| _d S )Nr   r™   r   rL   é   rs   ó    é    é(   Úcreator_proc)r@   rˆ   r}   Ztext_block_size_lengthr   r›   rP   r©   r|   Zcompression_literalsrK   r€   r±   r°   r¹   Zrle_compressionÚhasattrrš   )	r9   r“   r6   Ztext_block_sizer‹   Z	cname_rawZcompression_literalZclZoffset1r#   r#   r$   r_   ê  sN    



z,SAS7BDATReader._process_columntext_subheaderc                 C  sÒ   | j }||7 }|d|  d d }t|ƒD ]¢}|tj|d   tj }|tj|d   tj }|tj|d   tj }|  |tj¡}	|  |tj	¡}
|  |tj
¡}| j|	 }||
|
| … }| j |  |¡¡ q*d S )Nr—   é   rr   r   )r@   r¨   r}   Zcolumn_name_pointer_lengthZ!column_name_text_subheader_offsetZcolumn_name_offset_offsetZcolumn_name_length_offsetrˆ   Z!column_name_text_subheader_lengthZcolumn_name_offset_lengthZcolumn_name_length_lengthrP   rQ   r©   rš   )r9   r“   r6   r²   Zcolumn_name_pointers_countrª   Ztext_subheaderZcol_name_offsetZcol_name_lengthÚidxÚ
col_offsetZcol_lenZname_rawÚcnamer#   r#   r$   r`     sD    ÿþÿÿþÿÿþÿ ÿ ÿ
z,SAS7BDATReader._process_columnname_subheaderc           
      C  sÞ   | j }|d|  d |d  }t|ƒD ]²}|| tj ||d   }|d|  tj ||d   }|d|  tj ||d   }|  ||¡}	| j |	¡ |  |tj	¡}	| j
 |	¡ |  |tj¡}	| j |	dkrÒdnd¡ q&d S )Nr—   r»   rr   r   ó   dó   s)r@   r¨   r}   Zcolumn_data_offset_offsetZcolumn_data_length_offsetZcolumn_type_offsetrˆ   rV   r©   Zcolumn_data_length_lengthrU   Zcolumn_type_lengthrW   )
r9   r“   r6   r²   Zcolumn_attributes_vectors_countrª   Zcol_data_offsetZcol_data_lenZ	col_typesr�   r#   r#   r$   ra   8  s*    ÿÿþ
ýÿÿz2SAS7BDATReader._process_columnattributes_subheaderc                 C  s   d S r8   r#   r˜   r#   r#   r$   rc   R  s    z,SAS7BDATReader._process_columnlist_subheaderc                 C  sx  | j }|tj d|  }|tj d|  }|tj d|  }|tj d|  }|tj d|  }|tj d|  }	|  |tj	¡}
t
|
t| jƒd ƒ}|  |tj¡}|  |tj¡}|  |tj¡}t
|t| jƒd ƒ}|  |tj¡}|  |	tj¡}| j| }|  |||| … ¡}| j| }|  |||| … ¡}t| jƒ}t|| j| ||| j| | j| ƒ}| j |¡ | j |¡ d S )Né   r   )r@   r}   Z)column_format_text_subheader_index_offsetZcolumn_format_offset_offsetZcolumn_format_length_offsetZ(column_label_text_subheader_index_offsetZcolumn_label_offset_offsetZcolumn_label_length_offsetrˆ   Z)column_format_text_subheader_index_lengthÚminr|   rP   Zcolumn_format_offset_lengthZcolumn_format_length_lengthZ(column_label_text_subheader_index_lengthZcolumn_label_offset_lengthZcolumn_label_length_lengthrš   rS   r-   rQ   rW   rU   rR   r©   )r9   r“   r6   r²   Ztext_subheader_formatZcol_format_offsetZcol_format_lenZtext_subheader_labelZcol_label_offsetZcol_label_lenr�   Z
format_idxZformat_startZ
format_lenZ	label_idxZlabel_startZ	label_lenZlabel_namesZcolumn_labelZformat_namesZcolumn_formatZcurrent_column_numberÚcolr#   r#   r$   rb   V  s`    ÿÿ ÿ ÿ ÿ ÿ
ÿ
ÿ
ú	z(SAS7BDATReader._process_format_subheader)rŽ   r'   c                 C  sò   |d kr| j d k	r| j }n|d kr(| j}t| jƒdkrF|  ¡  tdƒ‚|dkr`| j| jkr`tƒ S t|| j| j ƒ}| j 	d¡}| j 	d¡}t
j||ftd�| _t
j|d| ft
jd�| _d| _t| ƒ}| |¡ |  ¡ }| jd k	rî| | j¡}|S )Nr   zNo columns to parse from filer¿   rÀ   ri   rr   )rG   r­   r|   rW   rh   r
   rX   r   rÂ   Úcountrk   r�   ÚobjectÚ_string_chunkÚzerosZuint8Ú_byte_chunkÚ_current_row_in_chunk_indexr   r{   Ú_chunk_to_dataframerN   Z	set_index)r9   rŽ   ZndÚnsÚpÚrsltr#   r#   r$   r{   �  s*    

zSAS7BDATReader.readc                 C  s¦   g | _ | j | j¡| _t| jƒdkr(dS t| jƒ| jkrf|  ¡  dt| jƒd›d| jd›d�}t|ƒ‚|  ¡  | j	t
jkr‚|  ¡  | j	t
jt
jt
jg kr¢|  ¡ S dS )Nr   Tz-failed to read complete page from file (read r   z of z bytes)F)rT   r[   r{   rŠ   rA   r|   rh   r"   rŸ   r¢   r}   r    r£   r¤   r¡   Ú_read_next_page)r9   Úmsgr#   r#   r$   rÎ   ¬  s$    ÿþzSAS7BDATReader._read_next_pagec                 C  st  | j }| j}t|| |ƒ}i }d\}}t| jƒD �]*}| j| }| j| dkrÚ| j|d d …f j| jd d�}	t	j
|	tj|d�||< | jrÐ| j| tjkr®t|| dƒ||< n"| j| tjkrÐt|| dƒ||< |d7 }q0| j| dk�r<t	j
| j|d d …f |d	�||< | j�r2| jd k	�r2|  || j¡||< |d7 }q0|  ¡  td
t| j| ƒ› �ƒ‚q0t|| j|dd�}
|
S )N)r   r   r¿   r   ri   )rj   rN   r   r   rÀ   )rN   zunknown column type F)rS   rN   Úcopy)rÉ   rX   r¨   r³   rQ   rW   rÈ   Úviewrƒ   r!   r+   rk   Zfloat64rE   rR   r}   Zsas_date_formatsr,   Zsas_datetime_formatsrÆ   rI   rH   Ú_decode_stringr   rh   r"   Úreprr   )r9   ÚnÚmÚixrÍ   ZjsZjbÚjr1   Zcol_arrZdfr#   r#   r$   rÊ   Å  s2    
 
 
z"SAS7BDATReader._chunk_to_dataframec                 C  s   |  | jp| j¡S r8   )ÚdecoderH   rO   ©r9   Úbr#   r#   r$   rÒ   ä  s    zSAS7BDATReader._decode_stringr4   )rÚ   r'   c                 C  s   | j r|  |¡S |S d S r8   )rJ   rÒ   rÙ   r#   r#   r$   rš   ç  s    
z#SAS7BDATReader._convert_header_text)NTTNNTTrB   )N)!r;   r<   r=   Ú__doc__r>   r:   ro   rp   rq   rh   re   r’   r‡   rˆ   r   rœ   rf   r�   rŸ   r£   r\   r]   r^   r_   r`   ra   rc   rb   r{   rÎ   rÊ   rÒ   rš   r#   r#   r#   r$   r?      sN   
        ö ?F
+-!7r?   ).rÛ   Ú
__future__r   Úcollectionsr   r   r   r„   Útypingr   Znumpyrk   Zpandas._typingr   r   r	   Zpandas.errorsr
   r   Zpandasr!   r   r   Zpandas.io.commonr   Zpandas.io.sas._byteswapr   r   r   r   r   Zpandas.io.sas._sasr   r   Zpandas.io.sas.sas_constantsÚioZsasZsas_constantsr}   Zpandas.io.sas.sasreaderr   r%   r,   r-   ÚIteratorr?   r#   r#   r#   r$   Ú<module>   s&   