a
    ==ic0F  ã                   @   sÞ   d dl mZ d dlmZ d dlmZ d dlmZ ddlmZ ddlmZ ddlm	Z	 d d	l
Z
d d	lZd d	lZd d	lZe ¡  e d
¡Ze ej¡ dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZG dd„ deƒZd	S )é    )Úabstractmethod)Ú	workspace)Útimeout_guard)Údata_parallel_modelé   )Ú
checkpoint)ÚModuleRegister©Ú
module_mapNÚAnyExpc                 C   sÚ   t  g d¢¡ | d d dks6| d d dks6J dƒ‚| d d dk}|rV| d d n
| d d }|rr| d d n
| d d }|r†d	nd
| d d< |o | d d | d d< || d d< || d d< i | d< i | d d< | S )N)Zcaffe2z--caffe2_log_level=2z--caffe2_gpu_memory_tracking=0ÚdistributedZnum_gpusr   Znum_cpuszCNeed to specify num_gpus or num_cpus to decide which device to use.Zfirst_cpu_idZfirst_gpu_idÚcpuZgpuÚdeviceÚmodel_paramÚcombine_spatial_bnÚnum_xpusÚfirst_xpu_idÚtemp_varÚmetrics_output)r   Z
GlobalInit)ÚoptsZtrainWithCPUr   Ú	first_xpu© r   úq/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/caffe2/contrib/playground/AnyExp.pyÚinitOpts   s2    ÿÿþÿ
ÿÿ
ÿÿ
r   c                   C   s   t tƒ d S ©N)ÚregisterModuleMapr
   r   r   r   r   ÚinitDefaultModuleMap6   s    r   c                 C   s   t  | ¡ d S r   )r   r   r	   r   r   r   r   :   s    r   c                 C   s   t  | d d ¡}| | ¡S )NÚinputZinput_name_py)r   Z	getModuleÚget_input_dataset)r   ZmyAquireDataModuler   r   r   ÚaquireDatasets>   s    r   c                 C   s   t  t| ¡S r   )r   ZconstructTrainerClassÚAnyExpTrainer)r   r   r   r   ÚcreateTrainerClassC   s    r!   c                 C   s   t  | |¡S r   )r   ÚoverrideAdditionalMethods)ZmyTrainerClassr   r   r   r   r"   G   s    r"   c                  O   s   t j| i |¤ŽS r   )r   Úinitialize_params_from_file)ÚargsÚkwargsr   r   r   r#   K   s    r#   c                   @   sp  e Zd Zdd„ Zedd„ ƒZedd„ ƒZedd„ ƒZd	d
„ Zdd„ Z	dd„ Z
dd„ Zedd„ ƒZdd„ Zedd„ ƒZdd„ Zdd„ Zedd„ ƒZedd„ ƒZdd „ Zd!d"„ Zed#d$„ ƒZed%d&„ ƒZed'd(„ ƒZed)d*„ ƒZd+d,„ Zed-d.„ ƒZed/d0„ ƒZed1d2„ ƒZed3d4„ ƒZed5d6„ ƒZed7d8„ ƒZed9d:„ ƒZ ed;d<„ ƒZ!d=d>„ Z"d?d@„ Z#dAdB„ Z$dCdD„ Z%dEdF„ Z&dGS )Hr    c                 C   s€  dd l }| ¡  | d¡}| |j¡ || _|| _d | _d | _d | _	d | _
i | _g | _g | _g | _g | _|d d | _|d d }|d d }t||| ƒ| _| jd d	 | jd d  | jd d
  | _| jd d | j | _t|d d ƒdkr0|d d d | _	t|d d ƒdkr0|d d d | _
|d d | _|d d | _|d d | _|d d | _| d t|ƒ¡¡ d S )Nr   r   r   r   r   r   r   Ú
epoch_iterZbatch_per_deviceZ
num_shardsZnum_train_sample_per_epochr   Zdatasetsé   r   Úshard_idÚstart_epochÚepochÚnum_epochs_per_flow_schedulezopts: {})ÚloggingÚbasicConfigÚ	getLoggerÚsetLevelÚDEBUGÚlogr   Útrain_datasetÚtest_datasetZtrain_dfZtest_dfÚmetricsÚplotsIngredientsÚrecord_epochsÚsamples_per_secÚsecs_per_trainr   ÚrangeÚxpusÚtotal_batch_sizeÚepoch_iterationsÚlenr(   r)   r*   Zepochs_to_runÚinfoÚformatÚstr)Úselfr   r,   r1   r   r   r   r   r   Ú__init__Q   sL    
ÿþÿÿÿzAnyExpTrainer.__init__c                 C   s   d S r   r   ©rA   r   r   r   r   r   ‚   s    zAnyExpTrainer.get_input_datasetc                 C   s   d S r   r   ©rA   r   r   r   Úget_model_input_fun†   s    z!AnyExpTrainer.get_model_input_func                 C   s   d S r   r   rD   r   r   r   Ú
init_modelŠ   s    zAnyExpTrainer.init_modelc                 C   sX   | j d d }|D ]@}|  |d ¡}|f d| j i|d ¤Ž}|  |d ||d ¡ qd S )NÚoutputr4   Zmeter_pyr   Zmeter_kargsÚnameÚis_train)r   ÚgetMeterClassÚ
add_metric)rA   r4   ÚmetricZ
meterClassZmeterInstancer   r   r   Úinit_metricsŽ   s
    zAnyExpTrainer.init_metricsc                 C   s   t  ||¡S r   )r   ZgetClassFromModule)rA   Z	meterNamer   r   r   rJ   —   s    zAnyExpTrainer.getMeterClassc                 C   s6   | j }i ||< ||| d< ||| d< g || d< d S )NÚ
calculatorrI   rG   )r4   )rA   rH   rN   rI   r4   r   r   r   rK   š   s
    zAnyExpTrainer.add_metricc                 C   s¤   | j }|sJ| j|d< | j|d< | j|d< | j ¡ D ]\}}|d ||< q2nV|d  | j¡ |d  | j¡ |d  | j¡ | j ¡ D ]\}}||  |d ¡ q„d S )NZepochsr7   r8   rG   )r   r6   r7   r8   r4   ÚitemsÚextend)rA   r   rL   Úvaluer   r   r   ÚextendMetricsOutput¡   s    


z!AnyExpTrainer.extendMetricsOutputc                 C   s   d S r   r   rD   r   r   r   Ú
init_plots°   s    zAnyExpTrainer.init_plotsc                 C   s8   | j }i }||d< ||d< ||d< ||d< | |¡ d S )NÚxÚx_titleÚysÚy_title)r5   Úappend)rA   rT   rU   rV   rW   r5   ZaPlotIngredientsr   r   r   Úadd_plot´   s    zAnyExpTrainer.add_plotc                 C   s   d S r   r   rD   r   r   r   Ú	init_logs½   s    zAnyExpTrainer.init_logsc                 C   s0   t | jd d | j| jd d  ƒ}t| j|ƒS )Nr&   Ú
num_epochsr+   )Úminr   r*   r9   )rA   Ziter_end_pointr   r   r   Úlist_of_epochsÁ   s    ÿÿzAnyExpTrainer.list_of_epochsc                 C   s   t d| jƒS )Nr   )r9   r<   rD   r   r   r   Úlist_of_epoch_itersÇ   s    z!AnyExpTrainer.list_of_epoch_itersc                 C   s   d S r   r   ©rA   r*   r   r   r   Úfun_per_epoch_b4RunNetÊ   s    z$AnyExpTrainer.fun_per_epoch_b4RunNetc                 C   s   d S r   r   r_   r   r   r   Úfun_per_epoch_aftRunNetÎ   s    z%AnyExpTrainer.fun_per_epoch_aftRunNetc              	   C   s2   t  d| j|  d|d ¡|d | jtdƒ¡| _d S )NTr   z-inf)r   Zsave_model_paramsÚtrain_modelÚgen_checkpoint_pathr   ÚfloatÚ
model_pathr_   r   r   r   r   Ò   s    þzAnyExpTrainer.checkpointc                 C   s&   |rd  |¡}nd}| jd d | S )Nzmodel_checkpoint_epoch{}.pklzmodel_final.pklrG   Zcheckpoint_folder)r?   r   )rA   Zis_checkpointr*   Úfilenamer   r   r   rc   ×   s    z!AnyExpTrainer.gen_checkpoint_pathc                 C   s   d S r   r   ©rA   r*   r&   r   r   r   Úfun_per_iter_b4RunNetâ   s    z#AnyExpTrainer.fun_per_iter_b4RunNetc                 C   s   d S r   r   rg   r   r   r   Úfun_per_iter_aftRunNetB4Testæ   s    z*AnyExpTrainer.fun_per_iter_aftRunNetB4Testc                 C   s   d S r   r   rg   r   r   r   Úfun_per_iter_aftRunNetAftTestê   s    z+AnyExpTrainer.fun_per_iter_aftRunNetAftTestc                 C   s   d S r   r   rC   r   r   r   Úfun_conclude_operatorî   s    z#AnyExpTrainer.fun_conclude_operatorc                 C   s   |   ¡  | j| _d S r   )rR   re   Zmodel_outputrD   r   r   r   ÚcreateMetricsPlotsModelsOutputsò   s    z-AnyExpTrainer.createMetricsPlotsModelsOutputsc                 C   s   d S r   r   rD   r   r   r   ÚassembleAllOutputsö   s    z AnyExpTrainer.assembleAllOutputsc                 C   s   d S r   r   ©rA   ÚmodelÚdatasetrI   r   r   r   Úgen_input_builder_funú   s    z#AnyExpTrainer.gen_input_builder_func                 C   s   d S r   r   rn   r   r   r   Úgen_forward_pass_builder_funþ   s    z*AnyExpTrainer.gen_forward_pass_builder_func                 C   s   d S r   r   rn   r   r   r   Úgen_param_update_builder_fun  s    z*AnyExpTrainer.gen_param_update_builder_func                 C   s   d S r   r   rn   r   r   r   Úgen_optimizer_fun  s    zAnyExpTrainer.gen_optimizer_func                 C   s   d S r   r   rn   r   r   r   Úgen_rendezvous_ctx
  s    z AnyExpTrainer.gen_rendezvous_ctxc                 C   s   d S r   r   rD   r   r   r   Úrun_training_net  s    zAnyExpTrainer.run_training_netc                 C   sT   | j d u rd S d}t |¡$ t | j j ¡ j¡ W d   ƒ n1 sF0    Y  d S )Ng     @Ÿ@)Ú
test_modelr   ZCompleteInTimeOrDier   ZRunNetÚnetÚProtorH   )rA   Útimeoutr   r   r   Úrun_testing_net  s
    
zAnyExpTrainer.run_testing_netc                 C   s   |   ¡  |  ¡  |  ¡  d S r   )rM   rS   rZ   rD   r   r   r   Úplanning_output  s    zAnyExpTrainer.planning_outputc                 C   s(   |   | j| jd¡ |   | j| jd¡ d S )NTF)Úprep_a_data_parallel_modelrb   r2   rw   r3   rD   r   r   r   Úprep_data_parallel_models   s    ÿÿz'AnyExpTrainer.prep_data_parallel_modelsc                 C   s0  |d u rd S t  d¡ | jd ur.|  |||¡nd }t  d¡ | jd urT|  |||¡nd }t  d¡ | jd d }tj||  |||¡|  |||¡||| j	|  
|||¡d| jd d | jd d || jd	 d
 dk| jd d d t  d¡ t |j¡ t  d¡ t  d |j ¡ ¡¡ t |j¡ t  d¡ d S )Nzin prep_a_data_parallel_modelz0in prep_a_data_parallel_model param_update done z-in prep_a_data_parallel_model optimizer done r   Úmax_concurrent_distributed_opsFZmemongerZ	cuda_ncclr   r   r   r   )Zinput_builder_funZforward_pass_builder_funZparam_update_builder_funZoptimizer_builder_funZdevicesZ
rendezvousZbroadcast_computed_paramsZoptimize_gradient_memoryZuse_ncclr   Z
cpu_devicer   z/in prep_a_data_parallel_model Parallelize done z.in prep_a_data_parallel_model RunNetOnce done zmodel.net.Proto() {}z-in prep_a_data_parallel_model CreateNet done )r1   r>   rs   rt   r   r   ZParallelizerq   rr   r:   ru   r   Z
RunNetOnceZparam_init_netr?   rx   ry   Z	CreateNet)rA   ro   rp   rI   Zparam_updateZ	optimizerZmax_opsr   r   r   r}   &  sJ    
ÿþ
ÿþ
ÿï

z(AnyExpTrainer.prep_a_data_parallel_modelc           	      C   sÈ   | j }|d d }|d d }|d d }|d urntj |¡r¸t d |¡¡ tj| j	|||ddd	\}}}nJ|d ur¸tj |¡r¸t d
 |¡¡ tj| j	|||d|d d d	\}}}t
 | j	¡ d S )Nr   Zcheckpoint_modelÚpretrained_modelr   r   zLoad previous checkpoint:{}TF)ro   Zweights_filer   r   Zbroadcast_computed_paramÚreset_epochzLoad pretrained model: {}r   r   )r   ÚosÚpathÚexistsr1   r>   r?   r   r#   rb   r   ZFinalizeAfterCheckpoint)	rA   r   Zprevious_checkpointr€   r   r)   Zprev_checkpointed_lrZ_best_metricZbest_metricr   r   r   ÚloadCheckpoint^  s<    ÿúÿ
	
úÿ
zAnyExpTrainer.loadCheckpointc              	   C   s¸  t  d t|ƒ¡¡ t  d t| ƒ¡¡ t  d t| ƒ¡¡ t  d | j¡¡ t  d t | j	¡¡¡ t  d t | j
¡¡¡ t  d t | j¡¡¡ | jd urºt  d t | j¡¡¡ nt  d	 t | j¡¡¡ t  d
 t | j¡¡¡ t  d t | j¡¡¡ |  ¡  |  ¡  |  ¡  |  ¡  |  ¡  |  ¡ D ]j}t  d |¡¡ |  |¡ |  ¡ D ]*}t ¡ | _|  ||¡ | jd urŠ|  ¡  |  ||¡ t ¡ | _||d d  dkrx| j| j }| j |¡ | j }|| }| j! |¡ |t"|ƒ| j#  | _$| j% | j$¡ | j&D ]@}| j&| }|d s,q|d  '¡  |d  |d  (¡ ¡ qt ¡ | _)t*d|d d ƒD ]@}	|  +¡  | j&D ]*}| j&| }|d rœq€|d  '¡  q€qnt ¡ | _,| j,| j) | _-| j& .¡ D ]*}|d räqÒ|d  |d  (¡ ¡ qÒd | j$| j/d d || j#|¡}
|
d ||d d | j-¡7 }
| j& 0¡ D ]"\}}|
d ||d d ¡7 }
qDt  d |
¡¡ |  1||¡ qZ|  2|¡ |  3|¡ q2|  4¡  |  5¡  |  ¡ S )Nz(in buildModelAndTrain, trainer_input: {}zcheck type self: {}zcheck self dir: {}zcheck self source: {}z(check self get_input_dataset methods: {}z+check self gen_input_builder_fun method: {}z2check self gen_forward_pass_builder_fun method: {}z2check self gen_param_update_builder_fun method: {}z'check self gen_optimizer_fun method: {}z(check self assembleAllOutputs method: {}z/check self prep_data_parallel_models method: {}zstart training epoch {}r&   Znum_train_iteration_per_testr   rI   rN   rG   Znum_test_iterz)epoch:{}/{} iter:{}/{} secs_per_train:{} r[   z-samples_per_sec:{} loop {} tests takes {} secz {}:{} éÿÿÿÿzIter Stats: {})6r1   r>   r?   r@   ÚtypeÚdirÚ__dict__ÚinspectÚ	getsourcer   rq   rr   rs   rt   rm   r~   rE   rF   r|   r…   r]   r`   r^   ÚtimeZiter_start_timerh   rb   rv   ri   Ziter_end_timer8   rX   r;   r7   rd   r<   Zfract_epochr6   r4   ÚAddZComputeZtest_loop_start_timer9   r{   Ztest_loop_end_timeZsec_per_test_loopÚvaluesr   rO   rj   r   ra   rk   rl   )rA   r   r*   r&   r8   Zsample_trainedr7   ÚkeyrL   Z
_test_iterZlogStrrQ   r   r   r   ÚbuildModelAndTrain  sº    
ÿ
ÿ
ÿ

ÿ
ÿ
ÿ
ÿ



ÿÿÿ








ÿ
þþ
z AnyExpTrainer.buildModelAndTrainN)'Ú__name__Ú
__module__Ú__qualname__rB   r   r   rE   rF   rM   rJ   rK   rR   rS   rY   rZ   r]   r^   r`   ra   r   rc   rh   ri   rj   rk   rl   rm   rq   rr   rs   rt   ru   rv   r{   r|   r~   r}   r…   r   r   r   r   r   r    O   sl   1


	
	














8!r    )Úabcr   Zcaffe2.pythonr   r   r   Ú r   r   r
   r‚   rŠ   rŒ   r,   r-   r.   r1   r/   r0   r   r   r   r   r!   r"   r#   Úobjectr    r   r   r   r   Ú<module>   s*   
