a Sic@sVddlmZddlmZmZddlZddlmZddlZddl Z ddl m Z m Z m Z mZmZmZmZmZmZmZddlZddlZddlmmZddlmZmZmZddlm Z m!Z!ddl"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)m*Z*e rdd l+m,Z,m-Z-d Z.Gddde)Z/GdddejZ0Gddde/Z1ddddZ2dddddZ3dS)) annotations)abc defaultdictN)StringIO) IO TYPE_CHECKING DefaultDictHashableIteratorListLiteralMappingSequencecast) ArrayLike ReadCsvBufferScalar)EmptyDataError ParserError)find_stack_level) is_integer) is_dict_like) ParserBaseparser_defaults)Index MultiIndexucseZdZdddfdd ZdddddZdWd d d d dZdddddZdXd d dddZdddddZddddZ d d!d"d d#d$d%Z d&dd'd(Z d&d&d)d*d+Z d&d,d-d.d/Z d&dd0d1Zd2d"dd3d4d5Zd"d6d7d8d9Zd:d:d;dd?Zd:d:d;d@dAZd:d2d2d:dBdCdDZd:d:d;dEdFZdddGdHZdIZdJdKdLdMdNZd:ddOdPdQZdYd d:d dRdSZd:d:dTdUdVZZS)Z PythonParserzReadCsvBuffer[str] | listNonefreturnc  st|d_g_d_d_|d_tjrBj_nfdd_t |d_ |d_ |d_ t j trtj _ |d _|d _|d _|d _|d _|d_|dpd_d_d|vr|d_|d_|d_|d_|d_t |tr2ttt|_nt|dsBJ|d_ !\}_"_#$|j%\_&_%_'}tj&_(j)sƈ*j&\}_(_&d_+j%dur|_%j durtt,t-j&_ .j&_/d}j0r1j j&}|_2t-jdkr.t3dt45j}jdurTd|d}nt45j} d| d|d}t46|_7dS)zN Workhorse function for processing nested list into DataFrame Nrskiprowscs |jvSN)r!)xself[/var/www/html/django/DPS/env/lib/python3.9/site-packages/pandas/io/parsers/python_parser.pyMz'PythonParser.__init__.. skipfooter delimiter quotechar escapechar doublequoteskipinitialspacelineterminatorquotingskip_blank_linesnamesFhas_index_namesverbose thousandsdecimalcommentreadlineTz'Only length-1 decimal markers supportedz^[\-\+]?[0-9]*(z [0-9]*)?([0-9]?(E|e)\-?[0-9]+)?$z^[\-\+]?([0-9]+z |[0-9])*()8super__init__databufposline_posr!callableskipfunc_validate_skipfooter_argr*r+r, isinstancestrr-r.r/r0r1r2Z names_passedr4r5r6r7r8listrr hasattr _make_reader _col_indices_infer_columnsnum_original_columns unnamed_cols_extract_multi_indexer_columns index_namescolumns col_names orig_names_has_complex_date_col_get_index_name_name_processedrangelen_validate_parse_dates_presence_parse_date_cols parse_dates_set_noconvert_dtype_columns_no_thousands_columns ValueErrorreescapecompilenum) r%rkwdsrO_rNZno_thousands_columnsr7regexr6 __class__r$r'r<=s                           zPythonParser.__init__IO[str] | ReadCsvBuffer[str]c sBjdustdkr$jr*tdGfdddtj}|}durT|_n}|ggd}j s~|sj d7_ }|ggd}qnt t t |}|d}j d7_ j d7_ t|}|j|_tjt||d}jt|tj|dd} nfd d } | } | _dS) Nr:z.MyDialect N) __name__ __module__ __qualname__r+r,r-r.r/r1r0r&r$r&r' MyDialectsrkr)dialectT)rlstrictc3s@}t}||VD]}||Vq&dSr")r9r]r_splitstrip)linepat)rsepr&r'_reads  z(PythonParser._make_reader.._read)r+rVr0r\csvDialectr9_check_commentsrBr?rr rEr@Sniffersniffreaderrr>extendrFr=) r%rrkZdiarplinesZ lines_strsniffedZline_rdrryrsr&)rr%rrr'rHs8  zPythonParser._make_readerNz int | NonezRtuple[Index | None, Sequence[Hashable] | MultiIndex, Mapping[Hashable, ArrayLike]])rowsr cCsz||}Wn(ty6|jr(g}n |Yn0d|_t|j}t|s||j}|||j |j |j \}}}| ||j }|||fSt|d}d} |jr|t|kr|d} |dd}||} || \} }|| } ||| \}} || | || \}} || | fS)NFrr:) _get_lines StopIteration _first_chunkcloserFrQrV_maybe_dedup_names_get_empty_meta index_colrNdtype_maybe_make_multi_index_columnsrPcount_empty_valsr4 _rows_to_cols_exclude_implicit_index _convert_data_do_date_conversions _make_index)r%r}contentrOr3indexcol_dictZ conv_columnsZcount_empty_content_vals indexnamerowalldatar=Z conv_dataresult_columnsr&r&r'reads@         zPythonParser.readzlist[np.ndarray]z8tuple[Mapping[Hashable, np.ndarray], Sequence[Hashable]])rr csR||j}d|jr t|jt||fddt|D|fS)Nrcs&i|]\}}|kr||qSr&r&).0inamerZ len_alldataoffsetr&r' 4sz8PythonParser._exclude_implicit_index..)rrQ_implicit_indexrVr_check_data_length enumerate)r%rr3r&rr'r&s   z$PythonParser._exclude_implicit_index)sizer cCs|dur|j}|j|dS)N)r}) chunksizer)r%rr&r&r' get_chunk9szPythonParser.get_chunkzMapping[Hashable, np.ndarray]zMapping[Hashable, ArrayLike])r=r c Cs||j}||j}i}i}t|jtr||jD]F}|j|}|j|}t|trh||jvrh|j|}|||<|||<q2n |j}|j}| ||||j ||Sr") _clean_mapping convertersrrD na_valuesdict na_fvaluesintrQ_convert_to_ndarraysr5) r%r=Z clean_conv clean_dtypesZclean_na_valuesZclean_na_fvaluescolna_valueZ na_fvaluer&r&r'rCs,        zPythonParser._convert_dataz9tuple[list[list[Scalar | None]], int, set[Scalar | None]]r c s,jd}d}t}d_jdurj}t|tttjfrft |dk}|rpt||ddg}n d}|g}g}t |D]l\}}z" } j |kr } qWnty} zdj kr|krsz/PythonParser._infer_columns...csh|] }|qSr&r&r) this_columnsr&r' r)z.PythonParser._infer_columns..zHNumber of passed names did not match number of header fields in the filez*Cannot pass names with multi-index columnscsg|] }|qSr&r&r)r3r&r'rr)csg|]}j|qSr&)prefixrr$r&r'rr))'r3set _header_lineheaderrDrFtuplenpndarrayrVr_buffered_liner@ _next_linermaprEjoinr\ _clear_bufferappendrmangle_dupe_colsrrrUrrgetupdaterr> TypeErrorusecols_handle_usecolsrIsortedrrA)r%rKZ clear_bufferrLrZhave_mi_columnsrOlevelhrrperrZjoimsgrccol_namecountsZcol_loop_orderrZold_col cur_countlcsicicZ unnamed_count first_lineZlen_first_data_rowncolsr&)r3r%rrr'rJfs&     $            "       $    "   zPythonParser._infer_columnszlist[list[Scalar | None]]zlist[Scalar | None]r)rO usecols_keyrKr c s|jdurt|jr$||j|ntdd|jDrt|dkrLtdg|jD]P}t|trz| |Wqty| |j|Yq0qV|qVn0fdd|jD}|rt j dt td |jfd d|D}t|_|S) zb Sets self._col_indices usecols_key is used if there are string usecols. Ncss|]}t|tVqdSr")rDrE)rur&r&r' Br)z/PythonParser._handle_usecols..r:z4If using multiple headers, usecols must be integers.csg|]}|kr|qSr&r&)rr)rKr&r'rRsz0PythonParser._handle_usecols..zkDefining usecols with out of bounds indices is deprecated and will raise a ParserError in a future version.) stacklevelcs"g|]}fddt|DqS)csg|]\}}|vr|qSr&r&)rrn col_indicesr&r'r_r)z;PythonParser._handle_usecols...)r)rcolumnrr&r'r^s)rrA_evaluate_usecolsanyrVr\rDrErr_validate_usecols_nameswarningswarn FutureWarningrrrI)r%rOrrKrZmissing_usecolsr&)rrKr'r3s>         zPythonParser._handle_usecolsz list[Scalar]cCs$t|jdkr|jdS|SdS)zH Return a line from buffer, filling buffer if required. rN)rVr>rr$r&r&r'res zPythonParser._buffered_line) first_rowr c Cs|s|St|dts|S|ds&|S|dd}|tkr>|S|d}t|dkr|d|jkrd}|d}|dd|d}|||}t||dkr|||dd7}n |dd}|g}||ddS)a- Checks whether the file begins with the BOM character. If it does, remove it. In addition, if there is quoting in the field subsequent to the BOM, remove it as well because it technically takes place at the beginning of the name, not the middle of it. rr:N)rDrE_BOMrVr,r) r%rZ first_eltZ first_row_bomstartquoteendnew_rowZ new_row_listr&r&r'_check_for_bomns(    zPythonParser._check_for_bombool)rpr cCs| ptdd|DS)z Check if a line is empty or not. Parameters ---------- line : str, array-like The line of data to check. Returns ------- boolean : Whether or not the line is empty. css|] }| VqdSr"r&rr#r&r&r'rr)z.PythonParser._is_line_empty..)all)r%rpr&r&r'_is_line_emptys zPythonParser._is_line_emptycCst|jtr||jr:|jt|jkr*q:|jd7_q zr||j|jgd}|jd7_|js||j|jds|rWqn"|jr| |g}|r|d}WqWq:t yt Yq:0q:n||jr|jd7_|jdusJt |jq|j |jdd}|jd7_|dur||gd}|jr\| |g}|rp|d}qrq||sr|rqrq|jdkr||}|jd7_|j||S)Nr:rrow_num)rDr=rFrBr?rVrvr2r_remove_empty_lines IndexErrorrnext_next_iter_linerr@r>r)r%rpretZ orig_liner&r&r'rsR          zPythonParser._next_linerE)rrr cCsJ|j|jjkrt|n.|j|jjkrFd|d}tj||ddS)a  Alert a user about a malformed row, depending on value of `self.on_bad_lines` enum. If `self.on_bad_lines` is ERROR, the alert will be `ParserError`. If `self.on_bad_lines` is WARN, the alert will be printed out. Parameters ---------- msg: str The error message to display. row_num: int The row number where the parsing error occurred. Because this row number is displayed, we 1-index, even though we 0-index internally. zSkipping line z: rgN) on_bad_linesBadLineHandleMethodERRORrWARNsysstderrwrite)r%rrbaser&r&r'_alert_malformeds   zPythonParser._alert_malformedzlist[Scalar] | None)rr c Csz,|jdusJt|j}t|ts(J|WStjy}zl|j|jjksZ|j|jj krt |}d|vsrd|vrvd}|j dkrd}|d|7}| ||WYd}~dSd}~00dS)a[ Wrapper around iterating through `self.data` (CSV source). When a CSV error is raised, we check for specific error messages that allow us to customize the error message displayed to the user. Parameters ---------- row_num: int The row number of the line being parsed. Nz NULL bytezline contains NULzNULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' insteadrzError could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).. ) r=rrDrFrtErrorrrrrrEr*r)r%rrperreasonr&r&r'rs(       zPythonParser._next_iter_linelist[list[Scalar]]r{r cCs|jdur|Sg}|D]t}g}|D]\}t|trD|j|vsD||jvrP||q"|d||j}t|dkrz||qq"||q|S)Nr)r8rDrErrfindrV)r%r{rrprlr#r&r&r'rv2s&     zPythonParser._check_commentscCsNg}|D]@}t|dks>t|dkrt|dtr>|dr||q|S)a Iterate through the lines and remove any that are either empty or contain only one whitespace value Parameters ---------- lines : list of list of Scalars The array of lines that we are to filter. Returns ------- filtered_lines : list of list of Scalars The same array of lines with the "empty" ones removed. r:r)rVrDrEror)r%r{rrpr&r&r'rGs     z PythonParser._remove_empty_linescCs |jdur|S|j||jddS)Nrr{searchreplace)r6_search_replace_num_columnsr%r{r&r&r'_check_thousandsas  zPythonParser._check_thousands)r{r r r c Csg}|D]t}g}t|D]X\}}t|trR||vsR|jrB||jvsR|j|s^||q||||q||q|Sr") rrDrEr[r`r rorr ) r%r{r r rrprrr#r&r&r'r is$  z(PythonParser._search_replace_num_columnscCs$|jtdkr|S|j||jddS)Nr7rr )r7rr r r&r&r'_check_decimal|s zPythonParser._check_decimalcCs g|_dSr")r>r$r&r&r'rszPythonParser._clear_bufferFzlist[Hashable]z@tuple[Sequence[Hashable] | None, list[Hashable], list[Hashable]])rOr c Cstt|}t|}|jdur"|j}n$z |}WntyDd}Yn0z |}Wntyhd}Yn0d}|dur$|j}|durt||j}|dur$|jdur$|dur$t|t||jkr$ttt||_|j dd|_ t |D]}| d|qt|}t||_|||fS|dkrTd|_ |jdurNtt||_d}n| ||j\}} |_|||fS)a Try several cases to get lines: 0) There are headers on row 0 and row 1 and their total summed lengths equals the length of the next line. Treat row 0 as columns and row 1 as indices 1) Look for implicit index: there are more columns on row 1 than row 0. If this is true, assume that row 1 lists index columns and row 0 lists normal columns. 2) Get index from the columns if it was listed. NrFr:T)rFrrrrrVrKrrUr>reversedinsertr_clean_index_names) r%rOrQrp next_lineZimplicit_first_colsrr index_namerbr&r&r'rSsP              zPythonParser._get_index_name)rr csj}jr|tj7}tdd|D}||krjdurjdurjrZjnd}g}t|}t|}g}|D]\}} t| } | |krtj r | } | dur| | nTj j j ksڈj j j krj|||} | | | fj j j krqqz| | qz|D]j\} } d|d| dd| } jrztjdkrzjtjkrzd }| d |7} | | dq"ttj||d j}jrjdusJjjrfd d t|D}nfdd t|D}|S)Ncss|]}t|VqdSr")rVrrowr&r&r'rr)z-PythonParser._rows_to_cols..Frz Expected z fields in line r:z, saw zXError could possibly be due to quotes being ignored when a multi-char delimiter is used.r) min_widthcs4g|],\}}|tjks,|tjvr|qSr&)rVrrrarr%r&r'rsz.PythonParser._rows_to_cols..csg|]\}}|vr|qSr&r&rrr&r'r(s)rKrrVrmaxrr*rrArrrrrr?r+r1rt QUOTE_NONErrFlibto_object_arrayTrI)r%rZcol_lenmax_lenZfootersZ bad_lines iter_contentZ content_lenrl actual_lenZnew_lrrrZzipped_contentr&rr'rst              zPythonParser._rows_to_colsc Cs^|j}d}|durPt|j|krB|jd||j|d}|_n|t|j8}|durt|jtr|jt|jkrzt|dur|j|jd}t|j}n |j|j|j|}|j|}||}||||_n.g}z|durvd}|j dur$|jdur$tt |j t t |j}t ||D]&}|jdusDJ| t |jq0t|}||}||nBd}|j|j|dd}|d7}|durz| |qzt|}Wn@tyt|}||}||t|dkrYn0|j|7_g|_n|}|jr.|d|j }||}|jrJ||}||}||S)Nrr:r)r>rVrDr=rFr?r_remove_skipped_rowsrzr!rrUrrrr*rvr2rrr) r%r}r{new_rowsnew_posZ rows_to_skiprbZ len_new_rowsrr&r&r'r~-sl"                 zPythonParser._get_lines)r%r cs jrfddt|DS|S)Ncs$g|]\}}|js|qSr&)rBr?)rrrr$r&r'rsz5PythonParser._remove_skipped_rows..)r!r)r%r%r&r$r'r$s  z!PythonParser._remove_skipped_rows)N)N)N)rhrirjr<rHrrrrrJrrrrrrrrvrrr rrrrSrr~r$ __classcell__r&r&rdr'r<s8tD2 #N2 680MWRrc @sbeZdZdZdddddddd d d d Zdddd dddZdddddddZd dddZdS)FixedWidthReaderz( A reader of fixed-width lines. Ndrfz(list[tuple[int, int]] | Literal['infer']z str | Nonezset[int] | Nonerr)rcolspecsr+r8r! infer_nrowsr cCs||_d|_|rd|nd|_||_|dkr>|j||d|_n||_t|jttfsht dt |j |jD]Z}t|ttfrt |dkrt|dt tjt dfrt|dt tjt dfsnt d qndS) Nz z infer)r+r!z;column specifications must be a list or tuple, input was a rrr:zEEach column specification must be 2 element tuple or list of integers)rbufferr+r8detect_colspecsr*rDrrFrtyperhrVrrinteger)r%rr*r+r8r!r+colspecr&r&r'r<s6     zFixedWidthReader.__init__z list[str])r+r!r cCsd|durt}g}g}t|jD]4\}}||vr:||||t||kr qVq t||_|S)a Read rows from self.f, skipping as specified. We distinguish buffer_rows (the first <= infer_nrows lines) from the rows returned to detect_colspecs because it's simpler to leave the other locations with skiprows logic alone than to modify them to deal with the fact we skipped some rows here as well. Parameters ---------- infer_nrows : int Number of rows to read from self.f, not counting rows that are skipped. skiprows: set, optional Indices of rows to skip. Returns ------- detect_rows : list of str A list containing the rows to read. N)rrrrrViterr-)r%r+r!Z buffer_rowsZ detect_rowsrrr&r&r'get_rowss    zFixedWidthReader.get_rowszlist[tuple[int, int]]c sdddjD}td|d}||}|s@tdttt|}t j |dt d}j dur|fd d|D}|D](}| |D]} d|| | <qqt |d} d | d <t || Adkd } tt| ddd | ddd } | S) NrcSsg|]}d|qS)\r&rr&r&r'rr)z4FixedWidthReader.detect_colspecs..z([^z]+)z(No rows from which to infer column widthr:)rcsg|]}|jdqS)r) partitionr8rr$r&r'rr)rr)rr+r]r_r3rrrrVrzerosrr8finditerrrrollwhererFzip) r%r+r! delimiterspatternr}r maskrmshiftededgesZ edge_pairsr&r$r'r.s"   "z FixedWidthReader.detect_colspecsrcs^jdur>ztjWqHty:d_tjYqH0n tjfddjDS)Ncs$g|]\}}||jqSr&)ror+)rZfrommtorpr%r&r'rr)z-FixedWidthReader.__next__..)r-rrrr*r$r&rBr'__next__s   zFixedWidthReader.__next__)Nr))N)r)N)rhrirj__doc__r<r3r.rCr&r&r&r'r(s &'r(c@s@eZdZdZdddddZddddd Zd d d d d ZdS)FixedWidthFieldParserzl Specialization that Converts fixed-width fields into DataFrames. See PythonParser for details. zReadCsvBuffer[str]rrcKs0|d|_|d|_tj||fi|dS)Nr*r+)popr*r+rr<)r%rrar&r&r'r<s  zFixedWidthFieldParser.__init__rfcCs"t||j|j|j|j|j|_dSr")r(r*r+r8r!r+r=)r%rr&r&r'rH sz"FixedWidthFieldParser._make_readerrrcCsdd|DS)z Returns the list of lines without the empty ones. With fixed-width fields, empty lines become arrays of empty strings. See PythonParser._remove_empty_lines. cSs"g|]}tdd|Dr|qS)css"|]}t|t p|VqdSr")rDrEro)rrr&r&r'rr)zGFixedWidthFieldParser._remove_empty_lines...)r)rrpr&r&r'rsz=FixedWidthFieldParser._remove_empty_lines..r&r r&r&r'rsz)FixedWidthFieldParser._remove_empty_linesN)rhrirjrDr<rHrr&r&r&r'rEs rErrcCstdd|DS)Ncss"|]}|dks|durdVqdS)rNr:r&)rvr&r&r'r"r)z#count_empty_vals..)sum)valsr&r&r'r!sr)r*r cCs$t|std|dkr td|S)a Validate the 'skipfooter' parameter. Checks whether 'skipfooter' is a non-negative integer. Raises a ValueError if that is not the case. Parameters ---------- skipfooter : non-negative integer The number of rows to skip at the end of the file. Returns ------- validated_skipfooter : non-negative integer The original input if the validation succeeds. Raises ------ ValueError : 'skipfooter' was not a non-negative integer. zskipfooter must be an integerrzskipfooter cannot be negative)rr\)r*r&r&r'rC%s rC)4 __future__r collectionsrrrtiorr]rtypingrrrr r r r r rrrnumpyrpandas._libs.lib_libsrpandas._typingrrr pandas.errorsrrpandas.util._exceptionsrpandas.core.dtypes.commonrpandas.core.dtypes.inferencerpandas.io.parsers.base_parserrrpandasrrrrr(rErrCr&r&r&r's>  0     Sv$