a yf b@s*dZddlZddlZddlZddlZddlZddlZddlmZddl Z ddl Z ddl Z ddlmZmZddlmZmZddlmZddlmZmZmZmZmZmZmZmZmZddl m!Z!m"Z"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)m*Z*ed dd dddddfddZ+GdddZ,GdddZ-dS)a Benchmark a YOLO model formats for speed and accuracy. Usage: from ultralytics.utils.benchmarks import ProfileModels, benchmark ProfileModels(['yolov8n.yaml', 'yolov8s.yaml']).profile() benchmark(model='yolov8n.pt', imgsz=160) Format | `format=argument` | Model --- | --- | --- PyTorch | - | yolov8n.pt TorchScript | `torchscript` | yolov8n.torchscript ONNX | `onnx` | yolov8n.onnx OpenVINO | `openvino` | yolov8n_openvino_model/ TensorRT | `engine` | yolov8n.engine CoreML | `coreml` | yolov8n.mlpackage TensorFlow SavedModel | `saved_model` | yolov8n_saved_model/ TensorFlow GraphDef | `pb` | yolov8n.pb TensorFlow Lite | `tflite` | yolov8n.tflite TensorFlow Edge TPU | `edgetpu` | yolov8n_edgetpu.tflite TensorFlow.js | `tfjs` | yolov8n_web_model/ PaddlePaddle | `paddle` | yolov8n_paddle_model/ NCNN | `ncnn` | yolov8n_ncnn_model/ NPath)YOLO YOLOWorld) TASK2DATA TASK2METRIC)export_formats) ARM64ASSETS IS_JETSONIS_RASPBERRYPILINUXLOGGERMACOSTQDM WEIGHTS_DIR)IS_PYTHON_3_12check_requirements check_yolo) safe_download) file_size) get_cpu_info select_devicez yolov8n.ptFcpuMbP?c sddldjj_djj_t|dd}t|ttfr>t |}t |j j ddd}g} t } t ttD]T\} \} } }}}d \}}z| d kr|jd ksJd nJ| d krtrtrJdn0| dvrtstsJdtrJdtrJd| dvrtrJd| dvr,t|tr,Jd| dvrJt|trJJd| dvrt|trhJd|rvJdtstsJd| dvrt|trJdd|jvr|sJdd |jvr|sJd!| d"kr|jp|j}|}n:|j|| |||dd#}t ||jd$}|t|vs4Jd%d&}|jd'ksV| d ksVJd(| dvshJd)| d*kstd+ksJd,| dvr|rJd-|jt d.|||d/|pt!|j}t"|j}|j#|d0|d|||dd1}|j$||j%d2}}t&d3||d4}| '| d5t&t(|d0t&|d6t&|d4|gWqpt)y}zj|rzt|t*uszJd7| d8|t+,d9| d8|| '| |t&t(|d0dddgWYd}~qpd}~00qpt-|d:j.| d;dd?gd@}t|jj/} dA| dB|dC|dDt | dEdF|dG }t+0|t1dHdIdJdKdL}|2|Wdn1sb0Y|rt|t3r||j4}|t5fdMdN|DsJdO|S)Pa Benchmark a YOLO model across different formats for speed and accuracy. Args: model (str | Path): Path to the model file or directory. data (str | None): Dataset to evaluate on, inherited from TASK2DATA if not passed. imgsz (int): Image size for the benchmark. half (bool): Use half-precision for the model if True. int8 (bool): Use int8-precision for the model if True. device (str): Device to run the benchmark on, either 'cpu' or 'cuda'. verbose (bool | float): If True or a float, assert benchmarks pass with given metric. eps (float): Epsilon value for divide by zero prevention. Returns: (pandas.DataFrame): A pandas DataFrame with benchmark results for each format, including file size, metric, and inference time. Examples: Benchmark a YOLO model with default settings: >>> from ultralytics.utils.benchmarks import benchmark >>> benchmark(model="yolov8n.pt", imgsz=640) rN xF)verboseZend2end)❌NZobbz.TensorFlow GraphDef not supported for OBB task z3Edge TPU export only supported on non-aarch64 Linux>rz9CoreML and TF.js export only supported on macOS and Linuxz5CoreML and TF.js export not supported on Raspberry Piz6CoreML and TF.js export not supported on NVIDIA Jetson>r#z#CoreML not supported on Python 3.12>r!z;YOLOWorldv2 TensorFlow exports not supported by onnx2tf yet>r"r> z,YOLOWorldv2 Paddle exports not supported yetz3End-to-end models not supported by PaddlePaddle yetz(Windows Paddle exports not supported yet> z*YOLOWorldv2 NCNN exports not supported yetrzinference not supported on CPUcudazinference not supported on GPU-)imgszformathalfint8devicer)taskz export failedu❎Zposez(GraphDef Pose inference is not supportedzinference not supportedr#Darwinz(inference only supported on macOS>=10.13zHEnd-to-end torch.topk operation is not supported for NCNN prediction yetzbus.jpg)r*r.r,)databatchr*Zplotsr.r,r-r inferenceu✅zBenchmark failure for : u#ERROR ❌️ Benchmark failure for )r.ZFormatu Status❔z Size (MB)zInference time (ms/im)ZFPS)columnsz Benchmarks complete for z on z at imgsz=z (z.2fzs)  zbenchmarks.logaignoreutf-8)errorsencodingc3s |]}|r|kVqdSN)Znotna).0xfloorpdX/var/www/html/django/DPS/env/lib/python3.9/site-packages/ultralytics/utils/benchmarks.py zbenchmark..z%Benchmark failure: metric(s) < floor )6ZpandasoptionsdisplayZ max_columnswidthr isinstancestrrrgetattrmodeltime enumerateziprvaluesr/r r rr r rrtypeZ ckpt_pathcfgexportplatformsystemZpredictr rrvalZ results_dictspeedroundappendr ExceptionAssertionErrorrwarningrZ DataFramenameinfoopenwritefloatarrayall)rPr2r*r,r-r.repsZ is_end2endyt0irar+suffixrgpuemojifilenameZexported_modelkeyresultsZmetricr[ZfpsedfsfZmetricsrFrCrG benchmark1s    &             0"8  . * (rvc@s>eZdZdZddZddZdddZed d Zd d Z d S)RF100BenchmarkzOBenchmark YOLO model performance across various formats for speed and accuracy.cCs g|_g|_d|_gd|_dS)zcInitialize the RF100Benchmark class for benchmarking YOLO model performance across various formats.NclassZimagestargets precisionZrecallmap50Zmap95)ds_names ds_cfg_listrfZ val_metrics)selfrFrFrG__init__szRF100Benchmark.__init__cCs$tdddlm}||d|_dS)a% Set Roboflow API key for processing. Args: api_key (str): The API key. Examples: Set the Roboflow API key for accessing datasets: >>> benchmark = RF100Benchmark() >>> benchmark.set_key("your_roboflow_api_key") roboflowr)Roboflow)api_keyN)rrrr)rrrrFrFrGset_keys  zRF100Benchmark.set_keydatasets_links.txtc Cs"tjdr tdtdfntdtdtdtdt|}|D]}zt d| \}}}}}|j ||d|} t| s|j|||dntd|j t| dWqTtyYqTYqT0qTWd n1s 0Y|j |jfS) a@ Parse dataset links and download datasets. Args: ds_link_txt (str): Path to the file containing dataset links. Examples: >>> benchmark = RF100Benchmark() >>> benchmark.set_key("api_key") >>> benchmark.parse_dataset("datasets_links.txt") zrf-100zultralytics-benchmarkszQhttps://github.com/ultralytics/assets/releases/download/v0.0.0/datasets_links.txtz/+r)Zyolov8zDataset already downloaded.z data.yamlN)ospathexistsshutilrmtreemkdirchdirrrcresplitstripr}r]rr workspaceprojectversiondownloadprintr~cwdr^) rZ ds_link_txtfileline_urlrrrZ proj_versionrFrFrG parse_datasets" *       ,zRF100Benchmark.parse_datasetcCs|t|}t|}Wdn1s(0Yd|d<d|d<t|d}t||Wdn1sn0YdS)z Fixes the train and validation paths in a given YAML file. Args: path (str): Path to the YAML file to be fixed. Examples: >>> RF100Benchmark.fix_yaml("path/to/data.yaml") Nz train/imagestrainz valid/imagesrZw)rcyaml safe_loadZ safe_dump)rrZ yaml_datarFrFrGfix_yamls ( zRF100Benchmark.fix_yamlc s|gd}t|}t|dWdn1s40Yt|dd}|}g} |D]`tfdd|Drxq\dttd d d d D| fd dDq\Wdn1s0Yd} t | dkrt d| D]} | ddkr| d} qnt ddd | Dd} t|d,}| |j |d| dWdn1sn0YdS)a Evaluate model performance on validation results. Args: yaml_path (str): Path to the YAML configuration file. val_log_file (str): Path to the validation log file. eval_log_file (str): Path to the evaluation log file. list_ind (int): Index of the current dataset in the list. Returns: (float): The mean average precision (mAP) value for the evaluated model. Examples: Evaluate a model on a specific dataset >>> benchmark = RF100Benchmark() >>> benchmark.evaluate("path/to/data.yaml", "path/to/val_log.txt", "path/to/eval_log.txt", 0) )u🚀u⚠️u💡r namesNr=)r?c3s|]}|vVqdSr@rF)rAsymbol)rrFrGrHrIz*RF100Benchmark.evaluate.. cSs|dkS)NrF)rZrFrFrGrIz)RF100Benchmark.evaluate..cSsg|]}|dqS)r:)rrArrrFrFrG rIz+RF100Benchmark.evaluate..c 3s^|]V}|vs&|dkrdvrdvrddddddd d Vqd S) rgz(AP)z(AR)rr1r6r7r#r%rxNrFr) class_namesentriesrFrGrHs  r1zThere's more dictsryrgr|zThere's only one dict rescSsg|] }|dqS)r|rF)rAresrFrFrGr-rIrr;r8r:) rcrr readlinesanyrlistfilterextendlenrrdr}) rZ yaml_pathZ val_log_fileZ eval_log_fileZlist_indZ skip_symbolsstreamrulinesZ eval_linesZmap_vallstrF)rrrrGevaluates2 ,  (    zRF100Benchmark.evaluateN)r) __name__ __module__ __qualname____doc__rrr staticmethodrrrFrFrFrGrws ! rwc@seZdZdZd#edd d Zd d Zd dZedddZ e d$ddZ d%ee dddZ d&ee dddZddZe dd Ze d!d"ZdS)' ProfileModelsa+ ProfileModels class for profiling different models on ONNX and TensorRT. This class profiles the performance of different models, returning results such as model speed and FLOPs. Attributes: paths (List[str]): Paths of the models to profile. num_timed_runs (int): Number of timed runs for the profiling. num_warmup_runs (int): Number of warmup runs before profiling. min_time (float): Minimum number of seconds to profile for. imgsz (int): Image size used in the models. half (bool): Flag to indicate whether to use FP16 half-precision for TensorRT profiling. trt (bool): Flag to indicate whether to profile using TensorRT. device (torch.device): Device used for profiling. Methods: profile: Profiles the models and prints the result. Examples: Profile models and print results >>> from ultralytics.utils.benchmarks import ProfileModels >>> profiler = ProfileModels(["yolov8n.yaml", "yolov8s.yaml"], imgsz=640) >>> profiler.profile() dr<TN)pathsc CsL||_||_||_||_||_||_||_|pDttj r@dnd|_dS)a[ Initialize the ProfileModels class for profiling models. Args: paths (List[str]): List of paths of the models to be profiled. num_timed_runs (int): Number of timed runs for the profiling. num_warmup_runs (int): Number of warmup runs before the actual profiling starts. min_time (float): Minimum time in seconds for profiling a model. imgsz (int): Size of the image used during profiling. half (bool): Flag to indicate whether to use FP16 half-precision for TensorRT profiling. trt (bool): Flag to indicate whether to profile using TensorRT. device (torch.device | None): Device used for profiling. If None, it is determined automatically. Notes: FP16 'half' argument option removed for ONNX as slower on CPU than FP32. Examples: Initialize and profile models >>> from ultralytics.utils.benchmarks import ProfileModels >>> profiler = ProfileModels(["yolov8n.yaml", "yolov8s.yaml"], imgsz=640) >>> profiler.profile() rrN) rnum_timed_runsnum_warmup_runsmin_timer*r,trttorchr.r( is_available) rrrrrr*r,rr.rFrFrGrMs!zProfileModels.__init__c Cs|}|stddSg}g}|D]}|d}|jdvrtt|}||}|jr|j j dkr| s|j d|j |j|j dd}|j d |j|j dd }n|jd kr$||}|}nq$|t|} |t|} |||j| | ||||j| | |q$|||S) z_Profiles YOLO models for speed and accuracy across various formats including ONNX and TensorRT.z'No matching *.pt or *.onnx files found.Nz.enginez.ptz.ymlz.yamlrZengineF)r+r,r*r.rZonnx)r+r*r.rz.onnx) get_filesr with_suffixrlrrNZfuserbrr.rUis_filerWr,r*get_onnx_model_infoprofile_tensorrt_modelprofile_onnx_modelr]generate_table_rowstemgenerate_results_dict print_table) rfiles table_rowsoutputr engine_filerP model_info onnx_filet_enginet_onnxrFrFrGprofilewsH      zProfileModels.profilecsg}|jD]dtr@gd}|fdd|Dq jdvrZ|tq |ttq tdt |ddt |DS)zGReturns a list of paths for all relevant model files given by the user.)z*.ptz*.onnxz*.yamlcs(g|] }tt|D]}|qqSrF)globrN)rAextrrrFrGrrIz+ProfileModels.get_files..rz Profiling: cSsg|] }t|qSrFr)rArrFrFrGrrI) rris_dirrrlr]rNrrsorted)rr extensionsrFrrGrs  zProfileModels.get_files)rcCsdS)zXExtracts metadata from an ONNX model file including parameters, GFLOPs, and input shape.)rrrrrF)rrrFrFrGrsz!ProfileModels.get_onnx_model_infor6rcCsnt|}t|D]V}t|t|}}|||||k||||k@}t|t|krdqj|}q|S)ziApplies iterative sigma clipping to data to remove outliers based on specified sigma and iteration count.)nprfrangemeanstdr)r2sigma max_itersrrrZ clipped_datarFrFrGiterative_sigma_clippings  $z&ProfileModels.iterative_sigma_clippingr)rrhc Cs |jrt|sdSt|}tj|j|jdtj }d}t dD]8}t }t |j D]}|||jddq\t |}qFt t|j|||j |jd}g} tt ||dD](}|||jdd} | | djd q|jt| d dd } t| t| fS) zaProfiles YOLO model performance with TensorRT, measuring average run time and standard deviation.)rrrrF)r*r2descrr4r6rr)rrrrrrandomrandr*astypefloat32rrQrmaxr\rrrr]r[rrfrr) rrrhrP input_dataelapsedr start_timenum_runs run_timesrqrFrFrGrs" $z$ProfileModels.profile_tensorrt_model)rrhcCstdddl}|}|jj|_d|_|j||dgd}|d}|j }t dd|j D }|rvd d |j |j fn|j } d |vrt j} nNd |vrt j} n>d |vrt j} n.d|vrt j} nd|vrt j} ntd|t jj| | } |j} |dj} d}td D]>}t}t|jD]}|| g| | iq"t|}q tt|j|||j|j }g}t!t||dD]6}t}|| g| | i|"t|dq|j#t $|ddd}t %|t &|fS)zeProfiles an ONNX model, measuring average inference time and standard deviation across multiple runs. onnxruntimerNr$ZCPUExecutionProvider) providerscss |]}t|to|dkVqdS)rN)rMint)rAdimrFrFrGrHrIz3ProfileModels.profile_onnx_model..r1rfloat16redoubleint64int32zUnsupported ONNX datatype rrr5r6r#r)'rrZSessionOptionsZGraphOptimizationLevelZORT_ENABLE_ALLZgraph_optimization_levelZintra_op_num_threadsZInferenceSession get_inputsrUrgshaper*rrrZfloat64rr ValueErrorrrrra get_outputsrrQrrunrr\rrrr]rrfrr)rrrhZortZ sess_optionssessZ input_tensorZ input_typeZdynamicZ input_shapeZ input_dtyperZ input_nameZ output_namerrrrrrFrFrGrsL    z ProfileModels.profile_onnx_modelc Csf|\}}}}d|dd|jd|ddd|ddd |ddd|ddd |d dd|dd S) zhGenerates a table row string with model performance metrics including inference times and model details.z| Z18sz | z | - | rz.1f±r1z ms | g.Az |)r*) r model_namerrrlayersparams gradientsflopsrFrFrGrs 6z ProfileModels.generate_table_rowcCs6|\}}}}||t|dt|ddt|dddS)zhGenerates a dictionary of profiling results including model name, parameters, GFLOPs, and speed metrics.rr)z model/namezmodel/parametersz model/GFLOPszmodel/speed_ONNX(ms)zmodel/speed_TensorRT(ms))r\)rrrrrrrr rFrFrGrs   z#ProfileModels.generate_results_dictcCstjrtjdnd}ddddtdd|d d d g}d d d d|Dd }d d dd|Dd }td|t||D] }t|qdS)zZPrints a formatted table of model profiling results, including speed and accuracy metrics.rZGPUZModelzsize
(pixels)zmAPval
50-95zSpeed
CPU (z) ONNX
(ms)zSpeed
z TensorRT
(ms)zparams
(M)zFLOPs
(B)|css|]}d|dVqdS)rNrFrAhrFrFrGrH8rIz,ProfileModels.print_table..css|]}dt|dVqdS)r)r6N)rr rFrFrGrH9rIz N)rr(rZget_device_namerjoinr)rrmheadersheader separatorrowrFrFrGr+s   zProfileModels.print_table)rrrrTTN)r6r)r)r)rrrrrrrrrNrrrrerrrrrrFrFrFrGr3s. *, 7 r).rrrrXrrrQpathlibrnumpyrZ torch.cudarrZ ultralyticsrrZultralytics.cfgrrZultralytics.engine.exporterrZultralytics.utilsr r r r r rrrrZultralytics.utils.checksrrrZultralytics.utils.downloadsrZultralytics.utils.filesrZultralytics.utils.torch_utilsrrrvrwrrFrFrFrGs<  ,   {