a Sic@sjdZddlZddlZddlmZddlmZddlZddl m Z m Z m Z m Z mZmZmZmZmZddlmZdd lmZmZmZmZmZmZdd l mZdd l mZdd l mZ dd l!m"Z"ddl#m$Z$ddl%m&Z&m'Z'e(e)dZ*e +j,Z-d$ddZ.d%ddZ/GdddZ0GdddZ1GdddZ2ddZ3e-rHGdddZ4Gd d!d!Z5Gd"d#d#eZ6dS)&z7 Implements support for high-level dataset access. N)warn)local) h5h5sh5th5rh5dh5ph5fdh5ds _selector)H5pyDeprecationWarning)HLObjectphil with_philEmptycached_propertyfind_item_type)filters) selections) selections2)Datatype)filename_decode)VDSmap vds_support FcCs|dur,t|ts,ddlm}|j||d}|dur\|durT|durLtdt|}|j}nJt|trl|fnt|}|durt j |t j dt j |jt j dkrt dt| tr| f} | dur| n|}t|trt|t s|f}t|trtdd t||Drd ||}t |t|tr4|j}|j}nT|durT|durTt d }n&|durp|durp|j}n t |}tj|dd }t|||| | fr|d urt d|dur| durd} d}|tvr| durtd|} d}t|pttj||||| ||| | || }| durjt|}|durVt|j}t j| |d} n t | } | | | durxd } | dvr|!| ntd|dur|"tj#tj$Bn(|d ur|"dn|durtd| durtdd | D} |dus |dur|pttj%}|dur2|&||durF|'|t|tr`t(t(j)}n t(*|| }t+j|j|||||d}|durt|ts|,t(j-t(j-||S)z+ Return a new low-level dataset identifier Nr)base)specified_dtypez-One of data, shape or dtype must be specifieddtypez%Shape tuple is incompatible with datacss"|]\}}|dur||kVqdSN).0dimchunkr#r#L/var/www/html/django/DPS/env/lib/python3.9/site-packages/h5py/_hl/dataset.py Esz make_new_dset..z^Chunk shape must not be greater than data shape in any dimension. {} is not compatible with {}z=f4)logicalFz1Chunked format required for given storage optionsTgzipzConflict in compression options)TFz(track_times must be either True or Falserz(track_order must be either True or Falsecss |]}|dur|ntjVqdSr"r UNLIMITED)r$mr#r#r'r()dcpldapl). isinstancerrarray_for_new_object TypeErrorshapeinttuplenumpyproduct ulonglong ValueErrorboolanyzipformatridr!r py_create_LEGACY_GZIP_COMPRESSION_VALSr fill_dcplr createDATASET_CREATEcheck_string_dtype string_dtypeencodingarrayset_fill_valueset_obj_track_timesset_attr_creation_orderCRT_ORDER_TRACKEDCRT_ORDER_INDEXEDDATASET_ACCESSset_efile_prefixset_virtual_prefixrNULL create_simpler writeALL)parentr6r!datanamechunks compressionshuffle fletcher32maxshapecompression_opts fillvalue scaleoffset track_timesexternal track_orderr0r1 efile_prefixvirtual_prefixallow_unknown_filterrZ tmp_shapeerrmsgtid string_infosiddset_idr#r#r' make_new_dset$s  *                               rmcKsd|dus|dur"|pttj}n|p(d}|dur<|||durN||tj|j||d}|S)z1 Return an existing low-level dataset identifier N)r1)r rErPrQrRr openrA)rWrYr1rerfkwdsrlr#r#r' open_dsets  rpc@s8eZdZdZddZddZddZdd Zd d Zd S) AstypeWrapperz7Wrapper to convert data on reading from a dataset. cCs||_t||_dSr")_dsetr9r!_dtype)selfdsetr!r#r#r'__init__szAstypeWrapper.__init__cCs|jj||jdSN) new_dtype)rr __getitem__rsrtargsr#r#r'ryszAstypeWrapper.__getitem__cCstdtdd|j|jj_|S)NztUsing astype() as a context manager is deprecated. Slice the returned object instead, like: ds.astype(np.int32)[:10]r)category stacklevel)rrrsrr_localastypertr#r#r' __enter__s  zAstypeWrapper.__enter__cGsd|jj_dSr")rrr~rrzr#r#r'__exit__szAstypeWrapper.__exit__cCs t|jS)zb Get the length of the underlying dataset >>> length = len(dataset.astype('f8')) lenrrrr#r#r'__len__szAstypeWrapper.__len__N) __name__ __module__ __qualname____doc__rvryrrrr#r#r#r'rqs  rqc@s*eZdZdZd ddZddZddZd S) AsStrWrapperz0Wrapper to decode strings on reading the datasetstrictcCs,||_|durt|jj}||_||_dSr")rrrrGr!rIerrors)rtrurIrr#r#r'rvs zAsStrWrapper.__init__csJj|}t|r$|jjStjfdd|jDtd |j S)Ncsg|]}|jjqSr#)decoderIr)r$brr#r' sz,AsStrWrapper.__getitem__..r ) rrr9isscalarrrIrrJflatobjectreshaper6)rtr{Z bytes_arrr#rr'rys  zAsStrWrapper.__getitem__cCs t|jS)z] Get the length of the underlying dataset >>> length = len(dataset.asstr()) rrr#r#r'rszAsStrWrapper.__len__N)r)rrrrrvryrr#r#r#r'rs rc@s,eZdZdZdZddZddZddZdS) FieldsWrapperzBWrapper to extract named fields from a dataset with a struct dtypeNcCs,||_t|tr||_|g}t|||_dSr")rrr2str extract_fieldreadtime_dtype read_dtype)rtruZ prior_dtypenamesr#r#r'rvs  zFieldsWrapper.__init__cCs*|jj||jd}|jdur&||j}|Srw)rrryrr)rtr{rXr#r#r'rys  zFieldsWrapper.__getitem__cCs t|jS)zh Get the length of the underlying dataset >>> length = len(dataset.fields(['x', 'y'])) rrr#r#r'rszFieldsWrapper.__len__)rrrrrrvryrr#r#r#r'rs rcsJjdurtd|D]}|jvrtd|qtfdd|DS)z=Make a NumPy compound dtype with a subset of available fieldsNz+Field names only allowed for compound typesz&Field %s does not appear in this type.csg|]}|j|dfqS)r)fields)r$rYbasetyper#r'rr/z"readtime_dtype..)rr<r9r!)rrrYr#rr'rs   rc@s(eZdZdZddZddZddZdS) CollectiveContextz$ Manages collective I/O in MPI mode cCs ||_dSr")rr)rtrur#r#r'rv!szCollectiveContext.__init__cCs|jjtjdSr")rr_dxpl set_dxpl_mpior MPIO_COLLECTIVErr#r#r'r$szCollectiveContext.__enter__cGs|jjtjdSr")rrrrr MPIO_INDEPENDENTrzr#r#r'r(szCollectiveContext.__exit__N)rrrrrvrrr#r#r#r'rsrc@s*eZdZdZd ddZddZddZdS) ChunkIteratorzD Class to iterate through list of chunks of a given dataset NcCs|j|_t|j}|js td|j|_|durdg}t|D]}|td|j|qd?Z$e d_d@dAZ%edBdCZ&e d`dDdEZ'e dFdGZ(dadHdIZ)dbdJdKZ*e dcdLdMZ+e dNdOZ,e-e.j/dPr"e dQdRZ0e-e.j/dSr>> double_precision = dataset.astype('f8')[0:100:2] )rq)rtr!r#r#r'r|szDataset.astypeNrcCs8t|j}|durtd|dur*|j}t|||dS)a(Get a wrapper to read string data as Python strings: >>> str_array = dataset.asstr()[:] The parameters have the same meaning as in ``bytes.decode()``. If ``encoding`` is unspecified, it will use the encoding in the HDF5 datatype (either ascii or utf-8). NzFdset.asstr() can only be used on datasets with an HDF5 string datatype)r)rrGr!r5rIr)rtrIrrjr#r#r'asstrs z Dataset.asstr _prior_dtypecCs|dur|j}t|||S)aQGet a wrapper to read a subset of fields from a compound data type: >>> 2d_coords = dataset.fields(['x', 'y'])[:] If names is a string, a single field is extracted, and the resulting arrays will have that dtype. Otherwise, it should be an iterable, and the read data will have a compound dtype. N)r!r)rtrrr#r#r'rs zDataset.fieldscCst|S)z3 Context manager for MPI collective reads & writes )rrr#r#r' collectiveszDataset.collectivecCs<ddlm}t||WdS1s.0YdS)z3 Access dimension scales attached to this dataset. r)DimensionManagerN)dimsrr)rtrr#r#r'rs z Dataset.dimscCs|jjS)z5Numpy-style attribute giving the number of dimensions)rArrr#r#r'ndimsz Dataset.ndimcCsTd|jvr|jdSt|jj}Wdn1s60Y|jrP||jd<|S)z1Numpy-style shape tuple giving dataset dimensionsr6N) _cache_propsrrAr6 _readonlyrtr6r#r#r'r6s  & z Dataset.shapecCs||dSr")resizerr#r#r'r6scCsFd|jvr|jdS|jr d}ntj|jtjd}|jrB||jd<|S)z3Numpy-style attribute giving the total dataset sizesizeNr )r _is_emptyr9prodr6intprrtrr#r#r'rs   z Dataset.sizecCs|j}|durdS|jj|S)zHNumpy-style attribute giving the raw dataset size as the number of bytesNr)rr!itemsizerr#r#r'nbytesszDataset.nbytescCs8d|jvr|jdSt|j}|jr4||jd<|S)z/Internal object for optimised selection of datar )rr ZSelectorrA get_spacer)rtZslrr#r#r'r s    zDataset._selectorcCs4d|jvr|jdSt|j}|jr0||jd<|S)z-Internal object for optimised reading of data _fast_reader)rr ReaderrAr)rtrdrr#r#r'rs     zDataset._fast_readercCs|jjS)z%Numpy dtype representing the datatype)rAr!rr#r#r'r!sz Dataset.dtypecCs |j}|tjkr|SdS)zDataset chunks (or None)N)_dcpl get_layoutr CHUNKED get_chunk)rtr0r#r#r'rZszDataset.chunkscCs dD]}||jvr|SqdS)zCompression strategy (or None))r+lzfszipN_filters)rtxr#r#r'r[s  zDataset.compressioncCs|j|jdS)z< Compression setting. Int(0-9) for gzip, 2-tuple for szip. N)rgetr[rr#r#r'r_szDataset.compression_optscCs d|jvS)zShuffle filter present (T/F)r\rrr#r#r'r\ szDataset.shufflecCs d|jvS)z"Fletcher32 filter is present (T/F)r]rrr#r#r'r]&szDataset.fletcher32cCs*z|jddWSty$YdS0dS)aScale/offset filter settings. For integer data types, this is the number of bits stored, or 0 for auto-detected. For floating point data types, this is the number of decimal places retained. If the scale/offset filter is not in use, this is None.rarN)rKeyErrorrr#r#r'ra,s zDataset.scaleoffsetcCsT|j}|dkrdSt}t|D]*}|j|\}}}|t|||fq$|S)zExternal file settings. Returns a list of tuples of (name, offset, size) for each external file entry, or returns None if no external files are used.rN)rget_external_countlistr get_externalrr)rtcountZext_listrrYoffsetrr#r#r'rc8s  zDataset.externalcCs2|j}|d}|dur dStdd|DS)zcShape up to which this dataset can be resized. Axes with value None have no resize limit. TNcss |]}|tjkr|ndVqdSr"r,r$rr#r#r'r(Qr/z#Dataset.maxshape..)rArget_simple_extent_dimsr8)rtspacerr#r#r'r^Gs   zDataset.maxshapecCs$tjd|jd}|j||dS)z*Fill value for this dataset (0 by default))rr r)r9zerosr!rget_fill_value)rtarrr#r#r'r`Ss zDataset.fillvaluecCs|jS)z9Get extent type for this dataset - SIMPLE, SCALAR or NULL)rArget_simple_extent_typerr#r#r' _extent_type[szDataset._extent_typecCs |jtjkS)zCheck if extent type is empty)rrrSrr#r#r'raszDataset._is_emptyF)readonlycslt|tjstd|t||j|_t t j |_ t |j|_||_i|_t|_d|j_dS)zJ Create a new Dataset object by binding to a low-level DatasetID. z%s is not a DatasetIDN)r2r DatasetIDr<superrvrAget_create_plistrr rE DATASET_XFERrr get_filtersrrrrr~r)rtbindr __class__r#r'rvfs    zDataset.__init__c Cst|jdurtd|dur|dkr4||jjksHtd|jjdz t|}WntyntdYn0t|j}|||<t |}|j |Wdn1s0YdS)aE Resize the dataset, or the specified axis. The dataset must be stored in chunked format; it can be resized up to the "maximum shape" (keyword maxshape) specified at creation time. The rank of the dataset cannot be changed. "Size" should be a shape tuple, or if an axis is specified, an integer. BEWARE: This functions differently than the NumPy resize() method! The data is not "reshuffled" to fit in the new shape; each axis is grown or shrunk independently. The coordinates of existing data are fixed. Nz$Only chunked datasets can be resizedrzInvalid axis (0 to %s allowed)rz2Argument must be a single int if axis is specified) rrZr5rArr<r7rr6r8 set_extent)rtraxisnewlenr#r#r'rvs    zDataset.resizecCs|}|tjkrtd|S)z The size of the first axis. TypeError if scalar. Limited to 2**32 on 32-bit systems; Dataset.len() is preferred. z>Value too big for Python's __len__; use Dataset.len() instead.)rsysmaxsize OverflowErrorrr#r#r'rs zDataset.__len__cCsJt2|j}t|dkr td|dWdS1s<0YdS)z The size of the first axis. TypeError if scalar. Use of this method is preferred to len(dset), as Python's built-in len() cannot handle values greater then 2**32 on 32-bit systems. rz'Attempt to take len() of scalar datasetN)rr6rr5rr#r#r'rs  z Dataset.lenccs:|j}t|dkrtdt|dD]}||Vq&dS)z Iterate over the first axis. TypeError if scalar. BEWARE: Modifications to the yielded data are *NOT* written to file. rz#Can't iterate over a scalar datasetN)r6rr5r)rtr6ir#r#r'rs  zDataset.__iter__cCs t||S)a Return chunk iterator. If set, the sel argument is a slice or tuple of slices that defines the region to be used. If not set, the entire dataspace will be used for the iterator. For each chunk within the given region, the iterator yields a tuple of slices that gives the intersection of the given chunk with the selection area. A TypeError will be raised if the dataset is not chunked. A ValueError will be raised if the selection region is invalid. )r)rtselr#r#r' iter_chunksszDataset.iter_chunkscCs$|jtjko"t|jtjtjfS)z+Is this dataset suitable for simple reading) rrSIMPLEr2rAget_typer TypeIntegerID TypeFloatIDrr#r#r' _fast_read_oks zDataset._fast_read_okcCst|tr|n|f}|dur*t|jdd}|jrZ|durZz|j|WStyXYn0|jr|dkst |dkr|dt urt |j St dtdd|D}|rt |dkr|d}td d|D}|j||d |S|dur|j }t|}t |dkrt|dtjrt|d|j}||jkr>t d t|d|j}t|}|durlt |Stj||d }|jdkr|St|} | |j| ||||S|jdkr|dkst |dkr|dt urtj|j|d S|jdkr||j} t !| |} | j"dur2tjd|d } ntj| j"|d } | D]\} } |j| | | |qF| j"durx| dS| Stj#|j||d } | j$dkrtj| j%|d Stj| j%|dd} t| j"} | j} |jj| | | ||j&d| jdkr| dS| S)a Read a slice from the HDF5 dataset. Takes slices and recarray-style field names (more than one is allowed!) in any order. Obeys basic NumPy rules, including broadcasting. Also supports: * Boolean "mask" array indexing Nrr#rrzEmpty datasets cannot be slicedcss|]}t|tr|VqdSr"r2rrr#r#r'r(r/z&Dataset.__getitem__..css|]}t|ts|VqdSr"rrr#r#r'r(r/rz+Region reference must point to this datasetr datasetC)orderdxpl)'r2r8getattrr~rrreadr5rrEllipsisrr!r<rrrBrRegionReference dereferencerA get_regionr guess_shaper9rrrrT select_allr6rsel2Z select_readmshapeselectnselect array_shaper)rtr{rxrmtypeobjrkr outZsid_outfspace selectionrmspacer#r#r'rysx             &         zDataset.__getitem__c st|tr|n|f}tdd|Dtdd|D}tjdurJttfvrJztj|d}WnHt yz"tj fdd|Djd}Wnt yYn0Yn0|jkr|j dkr*tj |j dd td }d d|tj|j dd tjd|j d fD|dd<ntj dgtd}||d <|}njjd ksjjdkrJt|tjr|jjdkrJjjdurJtdkrjjdurd jjvrt dd jjd d }d}n j}d}tj||jdd}|r|td |fg}||j dt|j t|j }njjdkrtjjdkrt|turtj|dtd}tj dd|jDjd|j }n*t|tjrdnjj}tj|d|d}jjdurXjjd}|j t| d} | |kr"td| |ftt|j|f} |j d t|j t|} n8td kr|j } jjdurtdfddD} t| d krd dd| D} t d| tdkr|jjdurt|j} t!tj"| #} | $%d d | nnfdd|jj&D}t!tj"|jj'} |D]@}t|jj|d } |jj|d}| $%||| qBn |j } d} t(j)j |d}|j*d krdS| d krH|j+d krHjjdurtd!j,r tj-j,tj.dtj-|j+tj.dkr tj |j+|jd}ntj |j+d |jd}||d"<|}|j } t/0|1| }|2| D]}j3j4|||| j5d#qbdS)$z Write to the HDF5 dataset from a Numpy array. NumPy's broadcasting rules are honored, for "simple" indexing (slices and integers). For advanced indexing, the shapes must match. css|]}t|tr|VqdSr"rrr#r#r'r(Jr/z&Dataset.__setitem__..css|]}t|ts|VqdSr"rrr#r#r'r(Kr/Nr csg|]}tj|dqS)r )r9rJr)vlenr#r'rUsz'Dataset.__setitem__..r)r6r!cSsg|]}|qSr#r#)r$rr#r#r'r\r/rOVzNo such field for indexing: %sTFr)r!rSutf-8)rr!cSsg|]}|dqS)r)encode)r$rr#r#r'rszWWhen writing to array types, last N dimensions have to match (got %s, but should be %s)z1Illegal slicing argument (not a compound dataset)csg|]}|jjvr|qSr#)r!rrrr#r'rr/z, css|]}d|VqdS)"%s"Nr#rr#r#r'r(r/z8Illegal slicing argument (fields %s not in dataset type)csg|]}|vr|qSr#r#r)rr#r'rr/rr#z5Scalar broadcasting is not supported for array dtypes.r)6r2r8rcheck_vlen_dtyper!bytesrr9asarrayr<rJremptyr6rrr:r;ravelkindndarraysubdtyperrrviewrGrIrrr5rBjoinrECOMPOUNDget_sizeinsert_errrr r rrZrfloat64rrTZ expand_shape broadcastrArUr)rtr{valtmpr!Z cast_compoundZ str_arraydtshpZvalshprr mismatchsubtype fieldnames fieldnamerrval2rrr#)rrtrr' __setitem__?s        "    &         zDataset.__setitem__cCst|jrtd|dur*t|j}nt|j||}|j}|durVt|j}nt|j|}||j D]}|jj ||||j dqpWdn1s0YdS)z Read data directly from HDF5 into an existing NumPy array. The destination array must be C-contiguous and writable. Selections must be the output of numpy.s_[]. Broadcasting is supported for simple indexing. z+Empty datasets have no numpy representationNr) rrr5rSimpleSelectionr6r rAr,rrr)rtdestrdest_selrrr#r#r' read_directszDataset.read_directcCst|jrtd|dur*t|j}nt|j|}|j}|durTt|j}nt|j||}||j D]}|jj ||||j dqpWdn1s0YdS)z Write data directly to HDF5 from a NumPy array. The source array must be C-contiguous. Selections must be the output of numpy.s_[]. Broadcasting is supported for simple indexing. z#Empty datasets cannot be written toNr) rrr5rr7r6r rAr,rrUr)rtsourcerr9rrr#r#r' write_directszDataset.write_directcCsFtj|j|dur|jn|d}tj|jtjddkr8|S|||S)z Create a Numpy array containing the whole dataset. DON'T THINK THIS MEANS DATASETS ARE INTERCHANGEABLE WITH ARRAYS. For one thing, you have to read the whole dataset every time this method is called. Nr r)r9rr6r!r:r;r:)rtr!rr#r#r' __array__ s  zDataset.__array__cCsX|s d}nJ|jdurd}n&tt|j}d|dkr:|nd}d||j|jjf}|S)Nzz ("anonymous")rr3/z&)rYppbasenamenormpathr6r!r)rtrnamestrrYr#r#r'__repr__s  zDataset.__repr__refreshcCs|j|jdS)z Refresh the dataset metadata by reloading from the file. This is part of the SWMR features and only exist when the HDF5 library version >=1.9.178 N)_idrErclearrr#r#r'rE)s zDataset.refreshflushcCs|jdS)a Flush the dataset data and metadata to the file. If the dataset is chunked, raw data chunks are written to the file. This is part of the SWMR features and only exist when the HDF5 library version >=1.9.178 N)rFrHrr#r#r'rH4sz Dataset.flushcCs|jtjkS)z"Check if this is a virtual dataset)rrr VIRTUALrr#r#r' is_virtual?szDataset.is_virtualcs.|jstd|jfddtDS)z5Get a list of the data mappings for a virtual datasetzNot a virtual datasetc s2g|]*}t||||qSr#)rget_virtual_vspaceget_virtual_filenameget_virtual_dsetnameget_virtual_srcspace)r$jr0r#r'rKs  z+Dataset.virtual_sources..)rJ RuntimeErrorrrget_virtual_countrr#rPr'virtual_sourcesEs   zDataset.virtual_sourcesr3cCst|j||dS)zMake this dataset an HDF5 dimension scale. You can then attach it to dimensions of other datasets like this:: other_ds.dims[0].attach_scale(ds) You can optionally pass a name to associate with this scale. N)r set_scalerFr*)rtrYr#r#r' make_scaleRs zDataset.make_scale)Nr)N)N)N)NN)NN)N)r3)7rrrrrrrMPIpropertyrrrrr6setterrrr rr!rZr[r_r\r]rarcr^r`rrrrvrrrrrrryr6r:r<r=rDhasattrr rrErHrrJrSrU __classcell__r#r#rr'rvs                              l         r)NNNNNNNNNNNNFNNNNNNF)NNN)7r posixpathr?rwarningsr threadingrr9r3rrrrr r r r r h5py_warningsrrrrrrrrrrrrr datatypercompatrvdsrr frozensetrrC get_configmpirVrmrprqrrrrrrr#r#r#r' sB  ,              I