a yfBy@sddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddlZddlZddlmZmZddlmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"ddl#m$Z$m%Z%m&Z&dd l'm(Z(m)Z)m*Z*dd l+m,Z,d Z-hd Z.hd Z/e0e1dd2dkZ3de.de/Z4ddZ5ddZ6ejdddZ7ddZ8ddZ9dd#d$Zd@d+d,Z?Gd-d.d.Z@dAd1d2ZAed3d4d5fd6d7ZBd8d9ZCd:d;ZDdS)BN) ThreadPool)Path) is_tarfile)ImageImageOps)check_class_names) DATASETS_DIRLOGGER NUM_THREADSROOT SETTINGS_FILETQDM clean_urlcolorstremojisis_dir_writeable yaml_load yaml_save) check_file check_fontis_ascii)download safe_download unzip_file)segments2boxeszJSee https://docs.ultralytics.com/datasets for dataset formatting guidance.> ZtiffZtifjpgZbmpZmpoZwebpZpfmZdngZpngjpeg> ZasfZmpegZmovtsZmp4ZmpgZaviZmkvZm4vZwmvZwebmZgif PIN_MEMORYTtruezSupported formats are: images: z videos: cs:tjdtjtjdtjfdd|DS)z0Define label paths as a function of image paths.imageslabelscs.g|]&}|dddddqS).rz.txt)joinrsplit.0xsasbR/var/www/html/django/DPS/env/lib/python3.9/site-packages/ultralytics/data/utils.py 0z#img2label_paths..)ossep)Z img_pathsr,r)r-img2label_paths-s&r2cCs@tdd|D}tt|}|d||S)z?Returns a single hash value of a list of paths (files or dirs).css&|]}tj|rtj|VqdSN)r0pathexistsgetsize)r'pr,r,r- 5r/zget_hash..)sumhashlibsha256strencodeupdater$ hexdigest)pathssizehr,r,r-get_hash3srD)imgcCsn|j}|jdkrjtt@|}|rL|dd}|dvrL|d|df}Wdn1s`0Y|S)z Returns exif-corrected PIL size.JPEGiN>r"r)rBformat contextlibsuppress ExceptionZgetexifget)rEsZexifZrotationr,r,r- exif_size;s   .rOc Csh|\\}}}d\}}}zt|}|t|}|d|df}|ddk|ddk@slJd|d|jtvsJd|jdt|jd vrt|d V} | d d | d krt t|j |dddd|d|d}Wdn1s0Yd}Wn>t yV} z$d}|d|d| }WYd} ~ n d} ~ 00||f|||fS)zVerify one image.)rrr9r"r image size <10 pixelszInvalid image format . rrrbrFdZ subsamplingqualityWARNING ⚠️ !: corrupt JPEG restored and savedN : ignoring corrupt image/label: )ropenverifyrOrIlower IMG_FORMATSFORMATS_HELP_MSGseekreadrexif_transposesaverL) argsim_fileclsprefixnfncmsgimshapefer,r,r- verify_imageHs(   ($   0*rsc s|\}}}}}}}dddddgdf\}} } } } } zFt|}|t|}|d|df}|ddk|ddk@sJd|d|jtvsJd|jd t|jd vr0t|d X}|d d | dkrt t|j |dddd|d|d} Wdn1s&0Yt j|rd} t|}dd| D}tdd|Dr|stjdd|Dtjd}dd|Dt|ddtfd}tj|tjd}Wdn1s0Yt|}|r|rf|jdd||ks8Jdd||d|ddddfd|dddd f}n:|jddksJd|jdd |ddddf}|dksJd!||dk|dksJd"||dk|dddf}||ks(Jd#t|d$|d%|dtj|dd&d'\}}t||kr||}rjfd(d|D|d|d)|t|d*} n*d} tjd|rd||ndftjd}n*d}tjd| rd||ndftjd}|rN|ddddfd||} |d krNt | d+dk| d,dkBd-d.!tj}tj| |d/gdd0} |ddddf}|||| || | | | f WSt"y}z.css|]}t|dkVqdS)rHN)rtr&r,r,r-r8zr/z%verify_image_label..cSsg|] }|dqS)rr,r&r,r,r-r.{r/ZdtypecSs,g|]$}tj|ddtjdddqS)r"NrvrW)nparrayfloat32reshaper&r,r,r-r.|r/rwzlabels require z columns eachzlabels require 5 columns, z columns detectedz,non-normalized or out of bounds coordinates znegative label values z Label class z exceeds dataset class count z. Possible class labels are 0-T)axisZ return_indexcsg|] }|qSr,r,r&segmentsr,r-r.r/: z duplicate labels removed).r).r"?).N)r}r^)#rr_r`rOrIrarbrcrdrerrfrgr0r4isfilestrip splitlinesanyrxryrz concatenater{rrtrpmaxminintuniquezeroswhereastyperL)rhriZlb_filerkZkeypointZnum_clsZnkptndimnmrlnermrn keypointsrorprqZlbclassesnlpointsZmax_cls_iZkpt_maskrrr,r~r-verify_image_labelasx  ($  0 00.&$$  (& *rr"cCsptj|tjd}tj|tjd}||jdddf}tj|||d|d||d|}}t |||fS)a Convert a list of polygons to a binary mask of the specified image size. Args: imgsz (tuple): The size of the image as (height, width). polygons (list[np.ndarray]): A list of polygons. Each polygon is an array with shape [N, M], where N is the number of polygons, and M is the number of points such that M % 2 = 0. color (int, optional): The color value to fill in the polygons on the mask. Defaults to 1. downsample_ratio (int, optional): Factor by which to downsample the mask. Defaults to 1. Returns: (np.ndarray): A binary mask of the specified image size with the polygons filled in. rvrrwrW)colorr") rxruint8asarrayint32r{rpcv2ZfillPolyresize)imgszpolygonsrdownsample_ratiomaskZnhnwr,r,r- polygon2masks rcstfdd|DS)a Convert a list of polygons to a set of binary masks of the specified image size. Args: imgsz (tuple): The size of the image as (height, width). polygons (list[np.ndarray]): A list of polygons. Each polygon is an array with shape [N, M], where N is the number of polygons, and M is the number of points such that M % 2 = 0. color (int): The color value to fill in the polygons on the masks. downsample_ratio (int, optional): Factor by which to downsample each mask. Defaults to 1. Returns: (np.ndarray): A set of binary masks of the specified image size with the polygons filled in. cs"g|]}t|dgqS)rw)rr{r&rrrr,r-r.r/z"polygons2masks..)rxry)rrrrr,rr-polygons2maskssrc Cstj|d||d|ft|dkr,tjntjd}g}g}tt|D]8}t|||dg|dd}|||| qJt |}t | }t ||}tt|D]0} || | d}||}tj |d| dd}q||fS)z!Return a (640, 640) overlap mask.rr"rvrw)rr)Za_minZa_max)rxrrtrrrangerr{appendr:rZargsortryZclip) rrrmasksZareasmssirindexrr,r,r-polygons2masks_overlaps$   r)r4returncstdptd}|s4Jddt|dkrRfdd|D}t|dksJddt|d ||d S) a  Find and return the YAML file associated with a Detect, Segment or Pose dataset. This function searches for a YAML file at the root level of the provided directory first, and if not found, it performs a recursive search. It prefers YAML files that have the same stem as the provided path. An AssertionError is raised if no YAML file is found or if multiple YAML files are found. Args: path (Path): The directory path to search for the YAML file. Returns: (Path): The path of the found YAML file. z*.yamlzNo YAML file found in ''r"csg|]}|jjkr|qSr,)stemr'rqr4r,r-r.r/z%find_dataset_yaml..zExpected 1 YAML file in 'z ', but found z. r)listglobrglobresolvert)r4filesr,rr-find_dataset_yamls  .rcst|}d}t|st|rFt|tddd}tt|}|jd}}t|dddD]L}|vrV|dksrdvrt t |d |d t d  dd<qVd vrd vrt t |dd vrd vrtd d krt t |dtd dd dd vr@ddtd Dd <ntd d <td d <t|pdptddjstd<dD]}|rt|tr|}|s |dr |dd}t||<nfdd|D|<qfdddD\}}|rddt|trj|n|gD}tdd|Dst|} d | d!d"d|Dd#d$} |r|rt | n| d%td&td$7} t| t} d} |d'r| d(rt|tdd)n8|d*rDt d+|d,t!"|} nt#|d-id.t$t| d/d0} | d1vrd2| d3t%d4tn d5| d6}t d7|d8t&t'd rd9nd:S);az Download, verify, and/or unzip a dataset if not found locally. This function checks the availability of a specified dataset, and if not found, it has the option to download and unzip the dataset. It then reads and parses the accompanying YAML data, ensuring key requirements are met and also resolves paths related to the dataset. Args: dataset (str): Path to the dataset or dataset descriptor (like a YAML file). autodownload (bool, optional): Whether to automatically download the dataset if not found. Defaults to True. Returns: (dict): Parsed dataset information and paths. r9TFdirZunzipdelete)Zappend_filename)trainvalr validation 'uE:' key missing ❌. 'train' and 'val' are required in all data YAMLs.uQWARNING ⚠️ renaming data YAML 'validation' key to 'val' to match YOLO format.namesrmuI key missing ❌. either 'names' or 'nc' are required in all data YAMLs.z 'names' length z and 'nc: z ' must match.cSsg|]}d|qS)class_r,)r'rr,r,r-r.&r/z%check_det_dataset..r4Z yaml_file)rrtestZminivalz../Ncsg|]}t|qSr,)r=rr&rr,r-r.;r/c3s|]}|VqdSr3)rMr&)datar,r-r8>r/z$check_det_dataset..)rrcSsg|]}t|qSr,)rrr&r,r,r-r.@r/css|]}|VqdSr3r5r&r,r,r-r8Ar/z Dataset 'u)' images not found ⚠️, missing path 'cSsg|]}|s|qSr,rr&r,r,r-r.Cr/rrz% Note dataset download directory is 'z'. You can update this in 'http.zip)urlrrbash zRunning z ...yaml(r"zs)>rNu success ✅ z , saved to boldzfailure u ❌zDataset download  z Arial.ttfzArial.Unicode.ttf)(rzipfile is_zipfilerrrrparentr SyntaxErrorrr infopoprtrrrrM is_absoluter isinstancer=r5 startswithrallrwarningr FileNotFoundErrortimeendswithr0systemexecroundrrr)dataset autodownloadfile extract_dirZnew_dirkr(rrNnamemtrdtr,)rr4r-check_det_datasetst    ** (         ,rr9c Cst|dr t|tddd}n&t|jdvrFt|}t|tddd}t|}|rZ|nt|}|st d|dt }t|dkrt j d td ddd nd |d }t||jddt |ddtd|d}t ||d}|dr|dn|dr*|dnd}|drD|dnd} |dkrd|sdt dn|dkr~| s~t dtdd|ddD} dd|dD} ttt| } ||| dD]\} } t| d d!| d"}| dur t |nd#d| d$D}t|}td%d&|D}|d'kr| dkrftt|d(| d)nt |d*|d+|d,nN|| krt |d*|d+|d-| d.| nt |d*|d+|d/q||| | | d0S)1a Checks a classification dataset such as Imagenet. This function accepts a `dataset` name and attempts to retrieve the corresponding dataset information. If the dataset is not found locally, it attempts to download the dataset from the internet and save it locally. Args: dataset (str | Path): The name of the dataset. split (str, optional): The split of the dataset. Either 'val', 'test', or ''. Defaults to ''. Returns: (dict): A dictionary containing the following keys: - 'train' (Path): The directory path containing the training set of the dataset. - 'val' (Path): The directory path containing the validation set of the dataset. - 'test' (Path): The directory path containing the test set of the dataset. - 'nc' (int): The number of classes in the dataset. - 'names' (dict): A dictionary of class names in the dataset. )zhttp:/zhttps:/TFr>rz.gzz.taru( Dataset not found ⚠️, missing path z, attempting download...Zimagenetrzdata/scripts/get_imagenet.sh)shellcheckz?https://github.com/ultralytics/assets/releases/download/v0.0.0/r)ruDataset download success ✅ (z.1fz s), saved to rrrrrNruIWARNING ⚠️ Dataset 'split=val' not found, using 'split=test' instead.uIWARNING ⚠️ Dataset 'split=test' not found, using 'split=val' instead.cSsg|]}|r|qSr,)is_dirr&r,r,r-r.r/z%check_cls_dataset..*cSsg|]}|r|jqSr,)rrr&r,r,r-r.r/rrr: ...cSs&g|]}|jddtvr|qSr"Nsuffixrarb)r'r4r,r,r-r.r/*.*cSsh|] }|jqSr,)r)r'rr,r,r- r/z$check_cls_dataset..rru :' no training images found ❌ z found z images in u( classes: WARNING ⚠️ no images foundu classes: ERROR ❌️ requires z classes, not u classes ✅ )rrrrmr)r=rrrrrrrrr rr subprocessrunr rrrrr5rtriterdirdict enumeratesorteditemsrrr)rrurdata_dirrrrNZ train_setZval_setZtest_setrmrrvrkrrlndr,r,r-check_cls_datasetZs\  "          ( rc@s@eZdZdZdddZeddZd d Zdd d Zd dZ dS)HUBDatasetStatsa A class for generating HUB dataset JSON and `-hub` dataset directory. Args: path (str): Path to data.yaml or data.zip (with data.yaml inside data.zip). Default is 'coco8.yaml'. task (str): Dataset task. Options are 'detect', 'segment', 'pose', 'classify'. Default is 'detect'. autodownload (bool): Attempt to download dataset if not found locally. Default is False. Example: Download *.zip files from https://github.com/ultralytics/hub/tree/main/example_datasets i.e. https://github.com/ultralytics/hub/raw/main/example_datasets/coco8.zip for coco8.zip. ```python from ultralytics.data.utils import HUBDatasetStats stats = HUBDatasetStats("path/to/coco8.zip", task="detect") # detect dataset stats = HUBDatasetStats("path/to/coco8-seg.zip", task="segment") # segment dataset stats = HUBDatasetStats("path/to/coco8-pose.zip", task="pose") # pose dataset stats = HUBDatasetStats("path/to/dota8.zip", task="obb") # OBB dataset stats = HUBDatasetStats("path/to/imagenet10.zip", task="classify") # classification dataset stats.get_json(save=True) stats.process_images() ``` coco8.yamldetectFc Cst|}td|d||_|jdkrHt|}t|}||d<nt|t|\}}}z0t|}d|d<t ||t ||}||d<Wn.t y} zt d| WYd} ~ n d} ~ 00t|dd|_ |j d |_ t|d t|d d |_||_dS) zInitialize class.z Starting HUB dataset checks for z....classifyr4r9zerror/HUB/dataset_stats/initNz-hubr r)rmr)rrr rtaskrr_unziprrrrLhub_dirim_dirrtrvaluesstatsr) selfr4rr unzip_dirrrrZ yaml_pathrrr,r,r-__init__s(         zHUBDatasetStats.__init__cCsVt|dsdd|fSt||jd}|sDJd|d|ddt|t|fS) zUnzip data.zip.rFNrzError unzipping z, z6 not found. path/to/abc.zip MUST unzip to path/to/abc/T)r=rrrrr)r4rr,r,r-rs  zHUBDatasetStats._unzipcCst||jt|jdS)z*Saves a compressed image for HUB previews.N)compress_one_imager rr)r rqr,r,r-_hub_opsszHUBDatasetStats._hub_opsc s>fdddD]}dj|<j|}|dur6qddt|dD}|sTqjdkrd d lm}|j|}t t |j  t }|jD]} || d d 7<qt ||d t |d |d dd|jDdj|<qd dlm} | j|jjd}tfddt|jt |ddD}t ||d d t |t t|d kd |d kd d fddt|j|jDdj|<q|rjjdddjd} td| dt| d} t j| Wdn1s0Y|r8ttj!jdddjS)z(Return dataset JSON for Ultralytics HUB.csjdkr|d}npjdvr2dd|dD}nRjdkrr|dj\}}}t|d|d|||fd }ntd jd t|d |}d d|DS)z:Update labels to integer class and 4 decimal place floats.rZbboxes>segmentZobbcSsg|] }|qSr,)flattenr&r,r,r-r.r/z._round..rZposerr"zUndefined dataset task=r#rjcSs,g|]$\}}t|dgdd|DqS)rcss|]}tt|dVqdS)N)rfloatr&r,r,r-r8r/zFHUBDatasetStats.get_json.._round...)r)r'crr,r,r-r.r/)rrprxrr{ ValueErrorzip)r!Z coordinatesnZnkrZzippedr r,r-_rounds    &z(HUBDatasetStats.get_json.._roundrNcSs&g|]}|jddtvr|qSrrrr,r,r-r.r/z,HUBDatasetStats.get_json..rrr) ImageFolderr")total per_class)rZ unlabelledrcSsg|]\}}t|j|iqSr,rrr'rrr,r,r-r.r/)Zinstance_statsZ image_statsr! YOLODataset)img_pathrrcs.g|]&}tj|dtjddqS)rjrm)Z minlength)rxZbincountrrrr)r'labelrr,r-r.sZ Statisticsrdesccs"g|]\}}t|j|iqSr,rr )rr,r-r.(r/Tparentsexist_okz stats.jsonzSaving rwrWF)indent sort_keys)"r rrMrrrZtorchvision.datasetsrrxrrtrrrZimgstolistultralytics.datar"ryr r!r:rrim_filesr mkdirr rrr_jsondumpdumps) r rgverboserur4rrrr(ror"Z stats_pathrqr,)rr r-get_jsonsT            .zHUBDatasetStats.get_jsonc Csddlm}|jjddddD]|}|j|dur6q ||j||jd}tt:}t| |j |j t ||dd D]}qxWdq 1s0Yq t d |j|jS) z$Compress images for Ultralytics HUB.rr!Tr'rN)r#rz imagesr%zDone. All images saved to )r.r"r r0rrMrr r imaprr/rtr r)r r"rurpoolrr,r,r-process_images6s  ("zHUBDatasetStats.process_imagesN)rrF)FF) __name__ __module__ __qualname____doc__r staticmethodrrr5r8r,r,r,r-rs   Jr2c Csz`t|}|t|j|j}|dkrH|t|j|t|j|f}|j|pR|d|ddWnty}zt d|d|t |}|j dd\}}|t||}|dkrt j|t||t||ft jd }t t|p||WYd}~n d}~00dS) at Compresses a single image file to reduced size while preserving its aspect ratio and quality using either the Python Imaging Library (PIL) or OpenCV library. If the input image is smaller than the maximum dimension, it will not be resized. Args: f (str): The path to the input image file. f_new (str, optional): The path to the output image file. If not specified, the input file will be overwritten. max_dim (int, optional): The maximum dimension (width or height) of the output image. Default is 1920 pixels. quality (int, optional): The image compression quality as a percentage. Default is 50%. Example: ```python from pathlib import Path from ultralytics.data.utils import compress_one_image for f in Path("path/to/dataset").rglob("*.jpg"): compress_one_image(f) ``` rrFT)r[optimizeu#WARNING ⚠️ HUB ops PIL failure rNrW) interpolation)rr_rheightwidthrrrgrLr rrZimreadrpZ INTER_AREAZimwriter=) rqZf_newZmax_dimr[rorrrZ im_heightZim_widthr,r,r-rFs " &rz coco8/images)g?g?rFc Cs t|}tdd|dD}t|}tdtjgd||d}gd}|D] }|j|rR|j| qRt d|d |t t |||d D]|\}} |rttt| gdrt|j||d 0} | d | |jd Wdq1s0YqdS)aL Automatically split a dataset into train/val/test splits and save the resulting splits into autosplit_*.txt files. Args: path (Path, optional): Path to images directory. Defaults to DATASETS_DIR / 'coco8/images'. weights (list | tuple, optional): Train, validation, and test split fractions. Defaults to (0.9, 0.1, 0.0). annotated_only (bool, optional): If True, only images with an associated txt file are used. Defaults to False. Example: ```python from ultralytics.data.utils import autosplit autosplit() ``` css(|] }|jddtvr|VqdSrrr&r,r,r-r8|r/zautosplit..rr)rr"rW)weightsr)zautosplit_train.txtzautosplit_val.txtzautosplit_test.txtzAutosplitting images from z!, using *.txt labeled images only)raz./rN)rrrrtrandomseedchoicesrr5unlinkr rr rr2r=r_write relative_toas_posix) r4rDZannotated_onlyrrindicestxtr(rrErqr,r,r- autosplitks rOcCs2ddl}|tjt|dd}||S)z1Load an Ultralytics *.cache dictionary from path.rNT)Z allow_pickle)gcdisablerxloadr=itemenable)r4rPcacher,r,r-load_dataset_cache_files rVcCst||d<t|jrX|r"|tt|||d|t |d|nt |d|jddS)z9Save an Ultralytics dataset *.cache dictionary x to path.versionz .cache.npyzNew cache created: uWARNING ⚠️ Cache directory z# is not writeable, cache not saved.N) rrr5rIrxrgr= with_suffixrenamer rr)rkr4r(rWr,r,r-save_dataset_cache_files rZ)r"r")r")r")T)r9)Nr>r?)ErJr;r1r0rFrrrZmultiprocessing.poolrpathlibrtarfilerrnumpyrxZPILrrZultralytics.nn.autobackendrZultralytics.utilsrr r r r r rrrrrrZultralytics.utils.checksrrrZultralytics.utils.downloadsrrrZultralytics.utils.opsrZHELP_URLrbZ VID_FORMATSr=getenvrarrcr2rDrOrsrrrrrrrrrrOrVrZr,r,r,r-sP    8  H    ] N %"