a
    j=icG                     @   s  d Z ddlZddlZddlZddlmZ ddlZddlZddlZddlZddl	m
Z
 ddlZddlZddlZddlZddlmZmZmZ ddlZddlZddlZddlmZmZ ddlmZ ddlm  mZ ddl Z!ddl m"Z"m#Z#m$Z$ ddl%m&Z' ddl(m  m)Z* dd	l+m,Z,m-Z- ej.d
Z/ej0dddd Z1dd Z2ej34ej35e6ddZ7eej34e7ddZ8ej0e8ddd Z9dd Z:dd Z;dd Z<ej.=dej>e;ddej>e!j?d dej>eje!j?ej@d!d"dej>eje!j?d#d!d$dej>eje!j?d%d!d&dgej.=d'e!j?e;gej.d(d)d* ZAd+d, ZBd-d. ZCej.=d/d0d1gd2d3 ZDej0d4d5 ZEG d6d7 d7ZFG d8d9 d9ZGej.=d:d;gd<e"g d=fd>e!jHIg d=g d?gfgd@dA ZJdBdC ZKej.=dDg dEdFdG ZLeMdHdIdJ ZNG dKdL dLejOZPdMdN ZQdOdP ZRdQdR ZSdSdT ZTej.=dUdVdWdUedXdYdYdZgd[d\ ZUd]d^ ZVd_d` ZWejXdadb ZYej.=dcejZej@gddde Z[dfdg Z\dS )ha<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    N)partial)Path)catch_warningsfilterwarningssimplefilter)get_lzma_fileis_platform_little_endian)import_optional_dependency)IndexSeriesperiod_range)DayMonthEndz1ignore:Timestamp.freq is deprecated:FutureWarningmodule)scopec                  C   sJ   ddl m}  t $ tddtd |  W  d    S 1 s<0    Y  d S )Nr   create_pickle_dataignorez The 'freq' argument in Timestamp)category)Z-pandas.tests.io.generate_legacy_storage_filesr   r   r   FutureWarningr    r   l/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/pandas/tests/io/test_pickle.pycurrent_pickle_data>   s    r   c                 C   s   t |trt||  d S |dr2t| | n`|dkrr|tju rT| tju spJ q| |ks`J | j|jksJ n t	td| dtj
}|| | d S )NZsp_	timestampZassert_Z_equal)
isinstancer
   tmassert_index_equal
startswithZassert_equalpdZNaTfreqgetattrZassert_almost_equal)resultexpectedtypZ
comparatorr   r   r   compare_elementN   s    


r$   datalegacy_pickle*z*.pickle)paramsc                 C   s
   || j S N)param)requestdatapathr   r   r   r&   d   s    c           	   	   C   s0  t  std tdd td t| }| D ]\}}| D ]\}}|| | }|dkrJ|dkrJt	|| |j
j|j
jksJ |j
jjrJ t	|dk|dk |j
j}|td td	ksJ |tjdd
 }t|tjsJ |tjdddksJ |tjdd }t|tjs0J |tjdddksJ qJ|dkr|dkrt|| t|jts|J |jt ksJ |jdksJ t|d	|d	 qJ|dkr|dv rt	|| qJ|dkr|dv rt|| qJt||| qJq8W d    n1 s"0    Y  d S )Nz"known failure on non-little endianT)recordr   Zseriestsr         )hours)daysr1   )nanoseconds)r2   r3   indexZperiodM)Zdt_tzcatframe)Zdt_mixed_tzsZ
cat_onecolZcat_and_float)r   pytestskipr   r   r   read_pickleitemsr   assert_series_equalr4   r   	normalizer   Z	Timedeltar   r   r   Zfreqstrshiftassert_frame_equalr$   )	r&   r%   r#   dvdtr!   r"   r   resr   r   r   test_picklesl   s@    

rC   c                 C   s>   t |d }tj| |dd W d    n1 s00    Y  d S Nwbprotocolopenpickledumpobjpathfhr   r   r   python_pickler   s    rQ   c                 C   sB   t | d$}|d t|W  d    S 1 s40    Y  d S )Nrbr   )rJ   seekrK   load)rO   rP   r   r   r   python_unpickler   s    
rU   pickle_writerpython)idZpandas_proto_defaultrG   Zpandas_proto_highest   Zpandas_proto_4   Zpandas_proto_5writerz5ignore:The 'freq' argument in Timestamp:FutureWarningc                 C   s  | }|  D ]\}}|  D ]\}}t }||| t|}	t|	|| t|}	t|	|| t|dd$}
||| |
d W d    n1 s0    Y  t|dd$}
t|
}	|
d W d    n1 s0    Y  t|	|| W d    q1 s0    Y  qqd S )NrE   moder   rR   )	r;   r   ensure_cleanr   r:   r$   rU   rJ   rS   )r   rV   r[   r%   r#   r@   rA   r"   rO   r!   handler   r   r   test_round_trip_current   s     



(
(r`   c                  C   s(   t  } t | jtj}t | | d S r)   )r   makeDataFrameround_trip_pathlib	to_pickler   r:   r?   dfr!   r   r   r   test_pickle_path_pathlib   s    rf   c                  C   s(   t  } t | jtj}t | | d S r)   )r   ra   Zround_trip_localpathrc   r   r:   r?   rd   r   r   r   test_pickle_path_localpath   s    rg   r#   ZsparseseriesZsparseframec                 C   sV   t t8 tdt tj| ddd| ddd W d   n1 sH0    Y  dS )	ap  

    Generated with

    >>> df = pd.DataFrame({"A": [1, 2, 3, 4], "B": [0, 0, 1, 1]}).to_sparse()
    >>> df.to_pickle("pandas/tests/io/data/pickle/sparseframe-0.20.3.pickle.gz",
    ...              compression="gzip")

    >>> s = df['B']
    >>> s.to_pickle("pandas/tests/io/data/pickle/sparseseries-0.20.3.pickle.gz",
    ...             compression="gzip")
    r   ior%   rK   z-0.20.3.pickle.gzgzipcompressionN)r   Zassert_produces_warningr   r   DeprecationWarningr   r:   )r,   r#   r   r   r   test_legacy_sparse_warning   s    
rm   c                   C   s   dt   dS )N__z	__.pickle)uuiduuid4r   r   r   r   get_random_path   s    rq   c                   @   sT   e Zd ZejZdd Zdd Zej	dg ddd Z
d	d
 Zdd Zdd ZdS )TestCompressionc           	   	   C   s  |d u rt || d S |dkr0t|d}n4|dkrHt|d}n|dkrtj|dtjd$}|	|t
j| W d    n1 s0    Y  n|dkrt|d^}tj|dd0}||t
j|}||| W d    n1 s0    Y  W d    n1 s0    Y  nF|d	kr6t |d}n.|d
krRtd|d}nd| }t||dvrt|dD}| |	|  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nri   wbz2ziprj   tarrR   r\   xzZzstdZ	zstandardrE   zUnrecognized compression type: )ru   rv   )shutilcopyfileri   rJ   rt   BZ2FilezipfileZipFileZIP_DEFLATEDwriteosrO   basenametarfile
gettarinfoaddfiler   r	   
ValueErrorread)	selfZsrc_path	dest_pathrk   frP   rv   tarinfomsgr   r   r   compress_file  s0    4
L



zTestCompression.compress_filec                 C   s  |}|d }|d }t |}t |}t  }|j||d t j||dH}	t|d}
|
|	  W d    n1 s0    Y  W d    n1 s0    Y  tj	|d d}t 
|| W d    n1 s0    Y  W d    n1 s0    Y  d S )N.compressed.rawrj   rE   )r   r^   ra   rc   decompress_filerJ   r~   r   r   r:   r?   )r   rk   rq   basepath1path2p1p2re   r   rP   df2r   r   r   test_write_explicit!  s    Jz#TestCompression.test_write_explicitrk   ) NonebadZ7zc              	   C   sr   t jtddP t|&}t }|j||d W d    n1 sF0    Y  W d    n1 sd0    Y  d S )NzUnrecognized compression type)matchrj   )r8   Zraisesr   r   r^   ra   rc   )r   rk   rq   rO   re   r   r   r   test_write_explicit_bad6  s    z'TestCompression.test_write_explicit_badc                 C   s  |}|| }|d }| j | }t|}t|}t }	|	| tj||dH}
t|d}|	|

  W d    n1 s0    Y  W d    n1 s0    Y  tj|d d}t|	| W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   rj   rE   )_extension_to_compressiongetlowerr   r^   ra   rc   r   rJ   r~   r   r   r:   r?   )r   compression_extrq   r   r   r   rk   r   r   re   r   rP   r   r   r   r   test_write_infer=  s    
Jz TestCompression.test_write_inferc           
   	   C   s   |}|d }|d }t |z}t |P}t  }|j|d d | j|||d tj||d}	t ||	 W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   r   rj   )r   r^   ra   rc   r   r   r:   r?   )
r   rk   rq   r   r   r   r   r   re   r   r   r   r   test_read_explicitS  s    z"TestCompression.test_read_explicitc              	   C   s   |}|d }|| }| j | }t|v}t|L}t }	|	j|d d | j|||d t	|}
t
|	|
 W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   rj   )r   r   r   r   r^   ra   rc   r   r   r:   r?   )r   r   rq   r   r   r   rk   r   r   re   r   r   r   r   test_read_inferf  s    
zTestCompression.test_read_inferN)__name__
__module____qualname__icomr   r   r   r8   markparametrizer   r   r   r   r   r   r   r   rr      s   
rr   c                   @   s&   e Zd Zejdg ddd ZdS )TestProtocolrH   )rF   r   r/   r0   c                 C   sZ   t |<}t  }|j||d t|}t || W d    n1 sL0    Y  d S )NrG   r   r^   ra   rc   r   r:   r?   )r   rH   rq   rO   re   r   r   r   r   	test_read  s
    
zTestProtocol.test_readN)r   r   r   r8   r   r   r   r   r   r   r   r     s   r   pickle_fileexcolsztest_py27.pkl)abcztest_mi_py27.pkl)ABCc                 C   s*   | ddd|}t |}t|j| d S )Nrh   r%   rK   )r   r:   r   r   columns)r,   r   r   rO   re   r   r   r   test_unicode_decode_error  s    
r   c               	   C   s   t  } t  }t| d}|| W d    n1 s<0    Y  t| d}t|}W d    n1 sp0    Y  t || W d    n1 s0    Y  d S )NrE   rR   )r   r^   ra   rJ   rc   r   r:   r?   )rO   re   rP   r!   r   r   r   test_pickle_buffer_roundtrip  s    
((r   mockurl)zhttp://url.comzftp://test.comzhttp://gzip.comc                    s   dd }G dd d t  R fdd}t  }|| | d| t|}t || W d    n1 sv0    Y  d S )Nc                 S   s>   t |d }tj| |dd W d    n1 s00    Y  d S rD   rI   rM   r   r   r   rQ     s    z3test_pickle_generalurl_read.<locals>.python_picklerc                   @   s:   e Zd ZddddZdd Zdd Zd	d
 Zdd ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponseNreturnc                 S   s.   t |d| _d|v r ddi| _n
ddi| _d S )NrR   ri   zContent-Encodingr   )rJ   fileheaders)r   rO   r   r   r   __init__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S   s   | S r)   r   r   r   r   r   	__enter__  s    z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W   s   |    d S r)   )close)r   argsr   r   r   __exit__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S   s
   | j  S r)   )r   r   r   r   r   r   r     s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S   s
   | j  S r)   )r   r   r   r   r   r   r     s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.close)r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s
   r   c                     s    S r)   r   )r   kwargsr   rO   r   r   mock_urlopen_read  s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   r^   ra   setattrr   r:   r?   )Zmonkeypatchr   rQ   r   re   r!   r   r   r   test_pickle_generalurl_read  s    


r   Zfsspecc                  C   sX   t  < d} t  }||  t| }t || W d    n1 sJ0    Y  d S )Nzmemory://afiler   )r   re   r!   r   r   r   test_pickle_fsspec_roundtrip  s    


r   c                   @   s   e Zd ZddddZdS )MyTzNr   c                 C   s   d S r)   r   r   r   r   r   r     s    zMyTz.__init__)r   r   r   r   r   r   r   r   r     s   r   c                  C   sD   t tdt f} t| }t|d | d  t|d ts@J d S )N)Zdtyper   r/   )r   objectr   r   round_trip_pickler<   r   )r"   r!   r   r   r   test_read_pickle_with_subclass  s    
r   c                 C   s   t  }t  *}|j|| d t| }W d   n1 s@0    Y  t }|j|| d |d |	 |ks| dv sJ t
j|| d}|d t || dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    rj   Nr   )ri   ru   rv   )r   ra   r^   rc   r   
read_bytesrh   BytesIOrS   getvaluer   r:   r?   )rk   re   rO   	referencebufferZread_dfr   r   r   %test_pickle_binary_object_compression  s    
*

r   c                 C   s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S   s   t | }t | | d S r)   )r   r   r?   )r7   	unpickledr   r   r   _test_roundtrip  s    
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataymdr7   r   r   r   r   +test_pickle_dataframe_with_multilevel_index  s    
r   c                  C   sB   t dddd} ttjt| | }t|}|jj	dks>J d S )Nz1/1/2011z1/1/2012r5   )r   )
r   r   nprandomZrandnlenr   r   r4   r   )Zprngr.   Znew_tsr   r   r   "test_pickle_timeseries_periodindex  s    
r   namei	  g     H@i     )r/   r0   c                 C   s$   t t j| d}|j| ks J d S )N)r   )r   r   ZmakeTimeSeriesr   )r   r   r   r   r   test_pickle_preserve_name&  s    r   c                 C   s   t | }t ||  d S r)   r   r   r<   )Zdatetime_seriesZunp_tsr   r   r   test_pickle_datetimes/  s    
r   c                 C   s   t | }t ||  d S r)   r   )Zstring_seriesZ
unp_seriesr   r   r   test_pickle_strings4  s    
r   c                  C   sf   t tddjdg } t| }|jjd jdks:J |jjd j	dksPJ t
|dg |  d S )Nabcr   r   r/   )r/   T)r   listZastypeZilocr   r   Z_mgrblocksndimshaper<   )ZserrB   r   r   r    test_pickle_preserves_block_ndim9  s
    
r   rH   c                 C   s@   t td}tt|j| |dtt j|d}t|| d S )Ni )rH   rk   rj   )	r   	DataFrameranger   rb   r   rc   r:   r?   )rH   rk   re   r!   r   r   r   %test_pickle_big_dataframe_compressionF  s    r   c                  C   s\   t jtdd} t| d}t|}W d    n1 s:0    Y  t }t	
|| d S )Nz1.2.4zempty_frame_v1_2_4-GH#42345.pklrR   )r   rO   joinlegacy_dirnamerJ   rK   rT   r   r   r   r?   )rO   fdre   r"   r   r   r   #test_pickle_frame_v124_unpickle_130Q  s
    (r   )]__doc__rt   datetime	functoolsr   globri   rh   r   pathlibr   rK   rx   r   ro   warningsr   r   r   r{   numpyr   r8   Zpandas.compatr   r   Zpandas.compat._optionalr	   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   r
   r   r   Zpandas._testingZ_testingr   Zpandas.io.commoncommonr   Zpandas.tseries.offsetsr   r   r   Z
pytestmarkZfixturer   r$   rO   r   dirname__file__r   filesr&   rC   rQ   rU   r   r*   rc   HIGHEST_PROTOCOLr`   rf   rg   rm   rq   rr   r   Z
MultiIndexZfrom_arraysr   r   r   Z
skip_if_nor   tzinfor   r   r   r   r   r   r   r   Zskip_array_manager_invalid_testr   DEFAULT_PROTOCOLr   r   r   r   r   r   <module>   s   



1


 



%
		



