a Sic*@sddlmZddlmZddlmZddlZddlZddlmZddl Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZmZmZddlZddlZddlmmZddlmmZ ddl!mm"Z"ddl!m#Z#ddl$m%Z%dd l&m'Z'm(Z(m)Z)dd l*m+Z+m,Z,dd l-m.Z.dd l/m0Z0dd l1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;mZ>ddl?m@Z@ddlAmBZBddlCmDZDddlEmFZFddlGmHZHmIZImJZJddlKmLZLddlMmNZOddlPmQZQe rddlRmSZSGdddZTd6ddZUdddejVdddddddddddddddddddddddddddddddddd eTjWjXddd!)ZYd7d"d#d$d%d&ZZd"d'd(d)Z[d*d+Z\d8d,d-d#d.d/d0Z]d1d2Z^d#d3d4d5Z_dS)9) annotations) defaultdictcopyN)Enum) TYPE_CHECKINGAnyCallable DefaultDictHashableIterableListMappingSequenceTuplecastfinaloverload) STR_NA_VALUES)parsing) ArrayLikeDtypeArgScalar) ParserError ParserWarning)find_stack_level)astype_nansafe) ensure_object is_bool_dtypeis_categorical_dtype is_dict_likeis_dtype_equalis_extension_array_dtype is_integeris_integer_dtype is_list_likeis_object_dtype is_scalaris_string_dtype pandas_dtype)CategoricalDtype)isna) algorithms) Categorical)Index MultiIndexensure_index_from_sequencesSeries) datetimes)generic_parser) DataFramec@s eZdZUGdddeZdZded<ded<ddd d Zd d d ddZddddZ e e ddddZ e dddddZ e dedddddZe d d dddZe dfd d d!d"d#d$Ze dgd%d&d'd(d)Ze d*d+Ze d,d-Zd.d/Ze dhdd1d2d3d4Ze did5dd6d7d8Ze d9d d:d;dd?Zd@dAZed1dBdCdDdEdFZed dGdHdDdIdFZdJdKdLdDdMdFZd dNddOdPdQZedRd d:dSdTdUZedVd dVdSdWdUZdXd dYdSdZdUZd[d\Zd]d^Zd_d`Z dkdadbdcddZ!dS)l ParserBasec@seZdZdZdZdZdS)zParserBase.BadLineHandleMethodrN)__name__ __module__ __qualname__ERRORWARNSKIPr?r?Y/var/www/html/django/DPS/env/lib/python3.9/site-packages/pandas/io/parsers/base_parser.pyBadLineHandleMethodPsrAFbool_implicit_index _first_chunkNonereturncCs|d|_d|_|dd|_|dd|_t|_d|_d|_ t |dd|_ g|_ |dd|_ |dd|_|dd|_|d |_|d |_|d d|_|d d |_t|dd|_|d|_|d|_|d|_|dd |_|dd|_|dd |_t|j |j|j|jd|_|d|_t|jddr|drbt d|drvt d|jdurt|jddrt!t"t#|jst#|jst dn |jdur|jdurt dd|_$d |_%|&|d\|_'|_(|d|j)j*|_+dS)Nnamesprefix index_col parse_datesF date_parserdayfirst keep_date_col na_values na_fvalues na_filterkeep_default_naTdtype converters true_values false_valuesmangle_dupe_colsinfer_datetime_format cache_dates)rLrMrXrYheader) allow_setsusecolsz;cannot specify usecols when specifying a multi-index headerz9cannot specify names when specifying a multi-index headerzLindex_col must only contain row numbers when specifying a multi-index headerz;Argument prefix must be None if argument header is not None on_bad_lines),getrH orig_namespoprIrJset unnamed_cols index_names col_names_validate_parse_dates_argrK_parse_date_colsrLrMrNrOrPrQrRrrSrTrUrVrWrXrY_make_date_converter _date_convrZr% ValueErrorallmapr#_name_processedrD_validate_usecols_argr\ usecols_dtyperAr<r])selfkwdsr?r?r@__init__Xsr           zParserBase.__init__zSequence[Hashable]r )columnsrGcst|jrtj|j}n(t|jr@tjdd|jD}ng}t|}dt fdd|D}|r|t d|dfdd |DS) a Check if parse_dates are in columns. If user has provided names for parse_dates, check if those columns are available. Parameters ---------- columns : list List of names of the dataframe. Returns ------- The names of the columns which will get parsed later if a dict or list is given as specification. Raises ------ ValueError If column to parse_date is not in dataframe. css*|]"}t|rt|ts|n|gVqdSN)r% isinstancetuple.0colr?r?r@ sz.z, cs"h|]}t|tr|vr|qSr?rtstrrvrrr?r@ sz.z+Missing column provided to 'parse_dates': ''cs*g|]"}t|ts|vr|n|qSr?rzrvr|r?r@ sz=ParserBase._validate_parse_dates_presence..) r rK itertoolschainvaluesr% from_iterablelistjoinsortedri)rorrZ cols_neededZ missing_colsr?r|r@_validate_parse_dates_presences,        z)ParserBase._validate_parse_dates_presencecCsdSrsr?ror?r?r@closeszParserBase.closecCs6t|jtp4t|jto4t|jdko4t|jdtS)Nr)rtrKdictrlenrr?r?r@_has_complex_date_cols    z ParserBase._has_complex_date_colint)irGcCst|jtr|jS|jdur(|j|}nd}|jdur:|n|j|}t|jrj||jkph|duoh||jkS||jvp|duo||jvSdSrs)rtrKrBrcrJr')rornamejr?r?r@_should_parse_datess      zParserBase._should_parse_datesz list | None)rc passed_namesc s&t|dkr|d|d|fSjdur.gttttjfsFgt|d} |j\}}}t|dt fdd|ddDst dfd d tt fd d|D}| }tD]}|||qtrfd d |D}ndgt|}d}||||fS)a Extract and return the names, index_names, col_names if the column names are a MultiIndex. Parameters ---------- header: list of lists The header rows index_names: list, optional The names of the future index passed_names: bool, default False A flag specifying if names where passed r8rNc3s|]}t|kVqdSrs)r)rwZ header_iter) field_countr?r@ry-z.r7z1Header rows must have an equal number of columns.cstfddtDS)Nc3s|]}|vr|VqdSrsr?)rwr)rsicr?r@ry1rzMParserBase._extract_multi_indexer_columns..extract..)ruranger)rrrr@extract0sz:ParserBase._extract_multi_indexer_columns..extractc3s|]}|VqdSrsr?rwr)rr?r@ry3rcs>g|]6}|ddur6|djvr6|dndqS)rN)rbr)icror?r@r:s z=ParserBase._extract_multi_indexer_columns..T)rrJrtrrunpndarrayrar`_clean_index_namesrjrziprrinsert) rorZrcr_rrrHZ single_icrdr?)rrrrorr@_extract_multi_indexer_columnss2      z)ParserBase._extract_multi_indexer_columns)rHrGcCs|jrt|}tt}t||j}t|D]\}}||}|dkr|d||<|rt|ts`J|dd|dd|f}n|d|}||}q:|||<|d||<q*|S)Nrr7r.) rWrrr_is_potential_multi_indexrJ enumeratertru)rorHcountsZis_potential_mirrxZ cur_countr?r?r@_maybe_dedup_namesGs   " zParserBase._maybe_dedup_namesNzSequence[Hashable] | NoneSequence[Hashable] | MultiIndex)rrrdrGcCs(t|r$ttt|}tj||dS|S)NrH)rrr rr/ from_tuples)rorrrdZ list_columnsr?r?r@_maybe_make_multi_index_columnsfsz*ParserBase._maybe_make_multi_index_columnszlist[Scalar] | Nonez4tuple[Index | None, Sequence[Hashable] | MultiIndex]) indexnamerowrGc Cst|jr|jsd}nf|js4|||}||}nH|jr||jsb|t||j\|_}|_d|_| ||}|j|dd}|rt |t |} |dusJ| |d| }| ||j }||fS)NTF)try_parse_dates) is_index_colrJr_get_simple_index _agg_indexrlrrrc_get_complex_date_indexr set_namesrrd) rodataZalldatarrrindexZ simple_indexrZ date_indexZcoffsetr?r?r@ _make_indexrs&     zParserBase._make_indexcCsldd}g}g}|jD]$}||}|||||qt|ddD]}|||jsH||qH|S)NcSs"t|ts|Std|ddS)NzIndex z invalid)rtr{rirxr?r?r@ixs z(ParserBase._get_simple_index..ixTreverse)rJappendrr`rC)rorrrr to_removeridxrr?r?r@rs    zParserBase._get_simple_indexc sjfdd}g}g}|jD]$}||}|||||qt|ddD]}|||qL|S)NcsLt|tr|Sdur&td|dtD]\}}||kr.|Sq.dS)Nz Must supply column order to use z as index)rtr{rir)icolrcrdr?r@ _get_names z5ParserBase._get_complex_date_index.._get_nameTr)rJrrr`remove) rorrdrrrrrrr?rr@rs     z"ParserBase._get_complex_date_indexcsttsSi}|jdus JD].\}}t|trN||jvrN|j|}|||<q(ttrt|jt|}|fdd|D|S)zconverts col numbers to namesNcsi|]}||qSr?r?rvmappingr?r@ rz-ParserBase._clean_mapping..) rtrr_itemsrrrakeysupdate)rorcleanrxvZremaining_colsr?rr@_clean_mappings    zParserBase._clean_mappingTr.)rrGcCsBg}||j}t|D]\}}|r:||r:||}|jrN|j}|j}n t}t}t |jt r|j dustJ|j |} | durt | |j|j|j \}}||j} d} d} |j durt | t r| |j |d} t |t r||j |du} | rt| p| } ||||B| \}}||q|j }t||}|SNF)rrTrrrhrQrOrPrartrrc_get_na_valuesrRrSr^r( _infer_typesrr0)rorrarraysrTrarr col_na_valuescol_na_fvaluescol_nameZ clean_dtypes cast_typeZindex_converter try_num_boolrrHr?r?r@rsD           zParserBase._agg_indexr)dctverbosec Cs i}|D] \}} |dur"dn ||d} t|trF||d} n|} |jrft||||j\} } ntt} } ||jvrt | t| | B}t | |t j | ||<q | durB| durtjd|dttdzt| | } Wn:ty"t | t|t j}t| | |} Yn0|j| t| | Bdd\}}nt| }|pVt| }| o`| }|| t| | B|\}}| rt|| rt| r|s|dkrzt| rtd|WnttfyYn0t | } |!|| |}|||<|r |r t"d|d |q |S) Nz5Both a converter and dtype were specified for column z# - only the converter will be used. stacklevelF)rrz$Bool column has NA values in column zFilled z NA values in column )#rr^rtrrQrrRrarfr,isinrputmasknanwarningswarnrrlib map_inferrirviewuint8map_infer_maskrr"r(r!rAttributeError TypeErrorr) _cast_typesprint)rorrOrPrrTdtypesresultrrZconv_frrrmaskZcvalsna_countZis_eaZis_str_or_ea_dtyperr?r?r@_convert_to_ndarrayss|         zParserBase._convert_to_ndarraysz list[int]zset[int]) col_indicesrHrGcs@t}|jdkrt|jnt|js0|jdvr6ndddfdd }t|jtr|jD]6}t|tr|D]}|||qtqb|||qbnt|jt r|j D]6}t|tr|D]}|||qq|||qnP|jr._set) rarnrr\callablertrKraddrrrJ)rorrHZnoconvert_columnsrvalkr?rr@_set_noconvert_dtype_columnsds4         z'ParserBase._set_noconvert_dtype_columnsc Cs0d}t|jjtjtjfrtdd|D}t||}|j ddd }|dkrxt |rh| tj }t ||tj||fS|rt|jrzt||d\}}Wn&ttfy|}t||}Yq0t| }n|}|jtjkrt||}|jtjkr(|r(tjt||j|jd\}}||fS)aU Infer types of values, possibly casting Parameters ---------- values : ndarray na_values : set try_num_bool : bool, default try try to cast values to numeric (first preference) or boolean Returns ------- converted : ndarray na_count : int rcSsg|]}t|ts|qSr?rz)rwrr?r?r@rrz+ParserBase._infer_types..rFr)rUrV) issubclassrStypernumberbool_arrayr,rastypesumr$float64rrr&rmaybe_convert_numericrirparserssanitize_objectsr+object_libopsmaybe_convert_boolasarrayrUrV)rorrOrrrrrr?r?r@rs6     zParserBase._infer_typesc Cs@t|rdt|to|jdu}t|s8|s8t|tt}t | }t j |||||jd}nt|rt|}|}z4t|r|j|||j|jdWS|j||dWSWn6ty}ztd|d|WYd}~n d}~00nRzt||ddd}Wn<ty:}z"td |d ||WYd}~n d}~00|S) aF Cast values to specified type Parameters ---------- values : ndarray cast_type : string or np.dtype dtype to cast values to column : string column name - used only for error reporting Returns ------- converted : ndarray N)rU)rSrUrVrSzExtension Array: zO must implement _from_sequence_of_strings in order to be used in parser methodsT)rskipnazUnable to convert column z to type )rrtr* categoriesr&rrrSr{r.uniquedropnar-_from_inferred_categories get_indexerrUr"r)construct_array_typer_from_sequence_of_stringsrVNotImplementedErrorri)rorrcolumnZ known_catscats array_typeerrr?r?r@rsJ   zParserBase._cast_typesr5z,tuple[Sequence[Hashable] | Index, DataFrame])rHrrGcCsdSrsr?rorHrr?r?r@_do_date_conversionsszParserBase._do_date_conversionszMapping[Hashable, ArrayLike]z7tuple[Sequence[Hashable], Mapping[Hashable, ArrayLike]]cCsdSrsr?rr?r?r@r&szSequence[Hashable] | Indexz(Mapping[Hashable, ArrayLike] | DataFramezKtuple[Sequence[Hashable] | Index, Mapping[Hashable, ArrayLike] | DataFrame]c Cs6|jdur.t||j|j|j|j||jd\}}||fS)N)rN)rK_process_date_conversionrhrJrcrNrr?r?r@r.s  zSequence[ArrayLike])rrrrGcCsz|jsvt|t|krv|rvt|do0|ddk}|t|dB}t|t|dkrdt|rddStjdtt ddS)aeChecks if length of data is equal to length of column names. One set of trailing commas is allowed. self.index_col not False results in a ParserError previously when lengths do not match. Parameters ---------- columns: list of column names data: list of array-likes containing the data column-wise. rr7NzkLength of header or names does not match length of data. This leads to a loss of data with index_col=False.r) rJrr&r+rrjrrrr)rorrrZ empty_strZempty_str_or_nar?r?r@_check_data_lengthBszParserBase._check_data_lengthz'set[int] | Callable[[Hashable], object])r\rHrGcCsdSrsr?ror\rHr?r?r@_evaluate_usecols_szParserBase._evaluate_usecolszset[str]cCsdSrsr?rr?r?r@rgsz2Callable[[Hashable], object] | set[str] | set[int]zset[str] | set[int]cs"trfddt|DSS)a  Check whether or not the 'usecols' parameter is a callable. If so, enumerates the 'names' parameter and returns a set of indices for each entry in 'names' that evaluates to True. If not a callable, returns 'usecols'. csh|]\}}|r|qSr?r?)rwrrr\r?r@r}zrz/ParserBase._evaluate_usecols..)rrrr?rr@rms cs0fdd|D}t|dkr,td||S)ai Validates that all usecols are present in a given list of names. If not, raise a ValueError that shows what usecols are missing. Parameters ---------- usecols : iterable of usecols The columns to validate are present in names. names : iterable of names The column names to check against. Returns ------- usecols : iterable of usecols The `usecols` parameter if the validation succeeds. Raises ------ ValueError : Columns were missing. Error message will list them. csg|]}|vr|qSr?r?rwrrr?r@rrz6ParserBase._validate_usecols_names..rz>Usecols do not match columns, columns expected but not found: )rri)ror\rHmissingr?rr@_validate_usecols_names}s z"ParserBase._validate_usecols_namescCsbd}|durZt|r|dfSt|s,t|tj|dd}|dvrJt|t|}||fS|dfS)as Validate the 'usecols' parameter. Checks whether or not the 'usecols' parameter contains all integers (column selection by index), strings (column by name) or is a callable. Raises a ValueError if that is not the case. Parameters ---------- usecols : list-like, callable, or None List of columns to use when parsing or a callable that can be used to filter a list of table columns. Returns ------- usecols_tuple : tuple A tuple of (verified_usecols, usecols_dtype). 'verified_usecols' is either a set if an array-like is passed in or 'usecols' if a callable or None is passed in. 'usecols_dtype` is the inferred dtype of 'usecols' if an array-like is passed in or None if a callable or None is passed in. z['usecols' must either be list-like of all strings, all unicode, all integers or a callable.NF)r)rrstring)rr%rir infer_dtypera)ror\msgrnr?r?r@rmsz ParserBase._validate_usecols_argc Cst|sd||fSt|}|s2dgt|||fSt|}g}t|}t|D]j\}}t|tr||t|D]&\}}||krr|||<||qqrqN||}||||qNt|D]$\}}t|tr||jvrd||<q|||fSrs) rrrrrtr{rrrb) rorrrJZcp_colsrcrrrrr?r?r@rs.       zParserBase._clean_index_nameszDtypeArg | Nonerc stt|s*|pttfddn*tt|}tddfdd|D|dusl|dusl|durvtg}nFfdd|D}t||d }| t |D]\}} ||qfd dD} || fS) NcsSrsr?r?) default_dtyper?r@rz,ParserBase._get_empty_meta..cSstSrs)objectr?r?r?r@r$rcs&i|]\}}t|r|n||qSr?)r#)rwrrr|r?r@rrz.ParserBase._get_empty_meta..Fcsg|]}tg|dqSrr1)rwr dtype_dictr?r@rrz.ParserBase._get_empty_meta..rcsi|]}|tg|dqSr&r1)rwrr'r?r@rs) rr r%rrrrr.r0sortrr`) rorrrJrcrSrrrncol_dictr?)rrr#r(r@_get_empty_metas(    zParserBase._get_empty_meta)F)N)N)T)FNN)T)N)"r9r:r;rrArC__annotations__rqrrrpropertyrrrrrrrrrrrrrrrrrrrrmrr,r?r?r?r@r6Osl  M=D   1\D 5@1&r6FTcsfdd}|S)Nc sdur`t|}z tjt|dddWSty\tjtj|ddYS0nz.tj|dd}t|t j rt d|WSt yz&tjtjt|dddWYSt yt g|RYYS0Yn0dS) Nignore)utcrMerrorsrXcache)rM)r2)r1r2z scalar parser)parserrM)r1) rconcat_date_colstools to_datetimerto_numpyrirrtdatetime Exceptionr4) date_colsstrsrrYrLrMrXr?r@ converter#sD      z'_make_date_converter..converterr?)rLrMrXrYr=r?r<r@rg s'rg"inferrstrict)) delimiter escapechar quotecharquoting doublequoteskipinitialspacelineterminatorrZrJrHrIskiprows skipfooternrowsrOrRrUrVrTrSrY thousandscommentdecimalrKrNrMrLr\ chunksizerencodingsqueeze compressionrWrXskip_blank_linesencoding_errorsr]error_bad_lineswarn_bad_linesr rB)r=rNcsfdd}g}i} |} t|}t} |dus:t|trB||fSt|tr|D]} t| sft| trt| tr| |vr| | } || rqP|t|| || <qPt || || \} }}| |vrt d| || | <| | | |qPnt|t rp|D]n\} } | |vr"t d| dt || || \}}}|| | <t| dkrX|| | d<| | | |q| | |||st| D]}||||q||fS)Ncs$ttr|vp"tto"|vSrs)rtr)colspecrJrcr?r@_isindexsz*_process_date_conversion.._isindexz New date column already in dict z Date column z already in dictr7r)rrartrBr'rurrr_try_convert_datesrirrrrrextendr`r) data_dictr=Z parse_specrJrcrrrNrXnew_colsnew_datar_r:rVnew_namerx old_namesrrr?rWr@r|sV             r)r3c st|}g}|D]D}||vr(||qt|trJ||vrJ|||q||qtdd|Dr~ttdjt|}nddd|D}fdd|D}||} || |fS)Ncss|]}t|tVqdSrs)rtrurwrr?r?r@ryrz%_try_convert_dates..rcSsg|] }t|qSr?)r{r`r?r?r@rrz&_try_convert_dates..cs"g|]}|vrt|qSr?)rrrr[r?r@rr) rarrtrrjrurkrr) r3rVr[rrZcolsetcolnamesrr^Zto_parseZnew_colr?rar@rYs  rYcCsJt|tr>||vr"||||fS|r0ttfSttfSn||fSdS)a Get the NaN values for a given column. Parameters ---------- col : str The name of the column. na_values : array-like, dict The object listing the NaN values as strings. na_fvalues : array-like, dict The object listing the NaN values as floats. keep_default_na : bool If `na_values` is a dict, and the column is not mapped in the dictionary, whether to return the default NaN values or the empty set. Returns ------- nan_tuple : A length-two tuple composed of 1) na_values : the string NaN values for that column. 2) na_fvalues : the float NaN values for that column. N)rtrrra)rxrOrPrRr?r?r@rs  rrzbool | Sequence[int] | None)rrrJrGcsDdusttrgtt|o@t|t o@tfdd|DS)a Check whether or not the `columns` parameter could be converted into a MultiIndex. Parameters ---------- columns : array-like Object which may or may not be convertible into a MultiIndex index_col : None, bool or list, optional Column or columns to use as the (possibly hierarchical) index Returns ------- bool : Whether or not columns could become a MultiIndex Nc3s$|]}|tvrt|tVqdSrs)rrtrurrJr?r@ryrz,_is_potential_multi_index..)rtrBrr/rj)rrrJr?rcr@rs rcCsBd}|dur>t|r(t|s>t|nt|ttfs>t||S)z Check whether or not the 'parse_dates' parameter is a non-boolean scalar. Raises a ValueError if that is the case. zSOnly booleans, lists, and dictionaries are accepted for the 'parse_dates' parameterN)r'ris_boolrrtrr)rKr"r?r?r@re s  rerFcCs|duo|duSrr?rr?r?r@r6sr)NFFT)F)N)` __future__r collectionsrrcsvr8enumrrtypingrrr r r r r rrrrrrrnumpyrpandas._libs.lib_libsrpandas._libs.opsopsrpandas._libs.parsersrrpandas._libs.tslibsrpandas._typingrrr pandas.errorsrrpandas.util._exceptionsrpandas.core.dtypes.astyperpandas.core.dtypes.commonrrrr r!r"r#r$r%r&r'r(r)pandas.core.dtypes.dtypesr*pandas.core.dtypes.missingr+ pandas.corer,pandas.core.arraysr-pandas.core.indexes.apir.r/r0pandas.core.seriesr2Zpandas.core.toolsr3r5Zpandas.io.date_convertersr4pandasr5r6rg QUOTE_MINIMALrAr<parser_defaultsrrYrrrerr?r?r?r@s    <    <        Y .6M%