a
    ==icw                     @   sL  d dl mZ d dlm  mZ d dlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZ d dlmZ d dlmZmZ d dlmZmZ d dlmZ d dlZd dlm Z m!Z! d dl"Z"d dl#Z#G d	d
 d
eeZ$G dd deeZ%G dd deeeZ&G dd deeeZ'G dd deeZ(G dd deeZ)G dd deeeZ*G dd deeZ+G dd deeZ,G dd deeeZ-G dd deeeZ.G dd  d eeeZ/G d!d" d"eeeZ0G d#d$ d$eeeZ1G d%d& d&eeeZ2G d'd( d(eeeZ3G d)d* d*eeZ4G d+d, d,eeeZ5G d-d. d.eZ6G d/d0 d0eZ7G d1d2 d2eZ8dS )3    )
caffe2_pb2N)	build_sgdbuild_multi_precision_sgd
build_ftrlbuild_gftrlbuild_wngradbuild_adagradbuild_adadelta
build_adambuild_yellowfinbuild_rms_propbuild_stormbuild_decay_adagradadd_weight_decaySgdOptimizer)UseOptimizer)OptimizerTestBaseLRModificationTestBase)core	workspace)TestCase)assert_allcloseassert_equalc                   @   s$   e Zd Zdd Zdd Zdd ZdS )TestLarsc                 C   s   t dd S Nzno sparse supportunittestZSkipTestself r   m/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/caffe2/python/optimizer_test.py
testSparse   s    zTestLars.testSparsec                 K   s   d| _ t|fddd|S )NF皙?      ?base_learning_ratelars	_skip_gpur   r   modelkwargsr   r   r    build_optimizer   s    zTestLars.build_optimizerc                 C   sX   |  | j | | j | jD ](}t|}tjj	t
dg|dd q*d S N      ?h㈵>Zatol
assertTrueget_auxiliary_parameterssharedassertFalselocalr   	FetchBlobnptestingr   arrayr   	optimizerparamZtensorr   r   r    check_optimizer   s
    
zTestLars.check_optimizerN__name__
__module____qualname__r!   r,   r>   r   r   r   r    r      s   r   c                   @   s   e Zd Zdd Zdd ZdS )TestMomentumSgdc                 K   s   d| _ t|fddd|S )NFr"   )r%   Zmomentumr'   r)   r   r   r    r,   '   s    zTestMomentumSgd.build_optimizerc                 C   sX   |  | j |  | j | jD ](}t|}tjjt	dg|dd q*d S r-   )
r2   r3   r4   r6   r   r7   r8   r9   r   r:   r;   r   r   r    r>   +   s
    
zTestMomentumSgd.check_optimizerNr@   rA   rB   r,   r>   r   r   r   r    rC   &   s   rC   c                   @   s   e Zd Zdd Zdd ZdS )TestSgdc                 K   s   d| _ t|fddi|S NFr%   r"   r'   r)   r   r   r    r,   4   s    zTestSgd.build_optimizerc                 C   sX   |  | j | | j | jD ](}t|}tjj	t
dg|dd q*d S r-   r1   r;   r   r   r    r>   8   s
    
zTestSgd.check_optimizerNrD   r   r   r   r    rE   3   s   rE   c                       s<   e Zd Zdd Zdd Zeej d fddZ	  Z
S )TestMultiPrecisionSgdc                 K   s   d| _ t|fddi|S rF   )r(   r   r)   r   r   r    r,   C   s    z%TestMultiPrecisionSgd.build_optimizerc                 C   sX   |  | j | | j | jD ](}t|}tjj	t
dg|dd q*d S r-   r1   r;   r   r   r    r>   I   s
    
z%TestMultiPrecisionSgd.check_optimizerzNo GPU supportc                    s   t t| tjj d S N)superrG   testGPUDenser   DataTypeZFLOAT16r   	__class__r   r    rJ   P   s    z"TestMultiPrecisionSgd.testGPUDense)r@   rA   rB   r,   r>   r   skipIfr   has_gpu_supportrJ   __classcell__r   r   rL   r    rG   @   s   rG   c                   @   s   e Zd Zdd Zdd ZdS )TestFtrlc                 K   s"   d| _ t|fd ddddd|S NTr.   r"           )ZenginealphabetaZlambda1Zlambda2)r(   r   r)   r   r   r    r,   V   s    zTestFtrl.build_optimizerc                 C   s>   |  | j | | j | jD ]}t| q*d S rH   r5   r3   r4   r2   r6   r   r7   r   r<   r=   r   r   r    r>   b   s    zTestFtrl.check_optimizerNrD   r   r   r   r    rQ   U   s   rQ   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )	TestGFtrlc                 C   s   t dd S r   r   r   r   r   r    r!   j   s    zTestGFtrl.testSparsec                 K   s"   d| _ t|fd ddddd|S rR   )r(   r   r)   r   r   r    r,   m   s    zTestGFtrl.build_optimizerc                 C   s>   |  | j | | j | jD ]}t| q*d S rH   rV   rW   r   r   r    r>   y   s    zTestGFtrl.check_optimizerNr?   r   r   r   r    rX   i   s   rX   c                   @   s   e Zd Zdd Zdd ZdS )TestAdagradc                 K   s   d| _ t|fddd|S )NFr.   r#   r$   r(   r   r)   r   r   r    r,      s    zTestAdagrad.build_optimizerc                 C   s>   |  | j | | j | jD ]}t| q*d S rH   rV   rW   r   r   r    r>      s    zTestAdagrad.check_optimizerNrD   r   r   r   r    rY      s   rY   c                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
TestRowWiseAdagradc                 K   s   d| _ t|fdddd|S )NTr.   r#   )r%   r&   rowWiserZ   r)   r   r   r    r,      s    z"TestRowWiseAdagrad.build_optimizerc                 C   s>   |  | j | | j | jD ]}t| q*d S rH   rV   rW   r   r   r    r>      s    z"TestRowWiseAdagrad.check_optimizerc                 C   s   t dd S Nzno dense supportr   r   r   r   r    	testDense   s    zTestRowWiseAdagrad.testDensec                 C   s   t dd S r]   r   r   r   r   r    rJ      s    zTestRowWiseAdagrad.testGPUDenseNr@   rA   rB   r,   r>   r^   rJ   r   r   r   r    r[      s   r[   c                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
TestRowWiseAdagradWithCounterc                 K   s    d| _ t|fddddd|S )NTr.   r#      )r%   r&   r\   Zcounter_halfliferZ   r)   r   r   r    r,      s    z-TestRowWiseAdagradWithCounter.build_optimizerc                 C   s   |  | j |  | j |  td td}tjj	t
dg|dd | jD ]}t| q^| jD ]}t| qxd S Noptimizer_iterationi  r/   r0   r2   r3   r4   r6   r   HasBlobr7   r8   r9   r   r:   r   r<   Ziteration_tensorr=   r   r   r    r>      s    
z-TestRowWiseAdagradWithCounter.check_optimizerc                 C   s   t dd S r]   r   r   r   r   r    r^      s    z'TestRowWiseAdagradWithCounter.testDensec                 C   s   t dd S r]   r   r   r   r   r    rJ      s    z*TestRowWiseAdagradWithCounter.testGPUDenseNr_   r   r   r   r    r`      s   r`   c                   @   s   e Zd Zdd Zdd ZdS )
TestWngradc                 K   s   d| _ t|fddi|S )NTr%   g      9@)r(   r   r)   r   r   r    r,      s    zTestWngrad.build_optimizerc                 C   s>   |  | j | | j | jD ]}t| q*d S rH   rV   rW   r   r   r    r>      s    zTestWngrad.check_optimizerNrD   r   r   r   r    rg      s   rg   c                   @   s   e Zd Zdd Zdd ZdS )	TestStormc                 K   s   d| _ t|fddi|S )NTr%          @)r(   r   r)   r   r   r    r,      s    zTestStorm.build_optimizerc                 C   s>   |  | j | | j | jD ]}t| q*d S rH   rV   rW   r   r   r    r>      s    zTestStorm.check_optimizerNrD   r   r   r   r    rh      s   rh   c                   @   s   e Zd Zdd Zdd ZdS )TestAdadeltac                 K   s   d| _ t|fddd|S )NFr.   gףp=
?)r%   Zdecay)r(   r	   r)   r   r   r    r,      s    zTestAdadelta.build_optimizerc                 C   s>   |  | j | | j | jD ]}t| q*d S rH   rV   rW   r   r   r    r>      s    zTestAdadelta.check_optimizerNrD   r   r   r   r    rj      s   rj   c                   @   s   e Zd Zdd Zdd ZdS )TestAdamc                 K   s   d| _ t|fddi|S rF   r(   r
   r)   r   r   r    r,      s    zTestAdam.build_optimizerc                 C   s   |  | j |  | j |  td td}tjj	t
dg|dd | jD ]}t| q^| jD ]}t| qxd S rb   rd   rf   r   r   r    r>      s    
zTestAdam.check_optimizerNrD   r   r   r   r    rk      s   rk   c                   @   s   e Zd Zdd Zdd ZdS )TestSmartDecayAdamc                 K   s$   d| _ d|d< t|fddd|S )NFrS   Zbeta1r"   T)r%   Zuse_smart_decayrl   r)   r   r   r    r,      s    z"TestSmartDecayAdam.build_optimizerc                 C   s   |  | j |  | j |  td t }|  tdd |D  | jD ]}t| qZ| jD ]}t| qtd S )Nrc   c                 s   s   | ]}| d V  qdS )Z
_last_seenN)endswith).0bnr   r   r    	<genexpr>       z5TestSmartDecayAdam.check_optimizer.<locals>.<genexpr>)	r2   r3   r4   r6   r   re   ZBlobsanyr7   )r   r<   Z
blob_namesr=   r   r   r    r>      s    z"TestSmartDecayAdam.check_optimizerNrD   r   r   r   r    rm      s   rm   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )TestDecayAdagradc                 K   s   d| _ t|fddi|S )NTr%   r.   )r(   r   r)   r   r   r    r,     s    z TestDecayAdagrad.build_optimizerc                 C   s   |  | j |  | j |  td td}tjj	t
dg|dd | jD ]}t| q^| jD ]}t| qxd S rb   rd   rf   r   r   r    r>   
  s    
z TestDecayAdagrad.check_optimizerc                 C   s   t dd S r   r   r   r   r   r    r!     s    zTestDecayAdagrad.testSparseNr@   rA   rB   r,   r>   r!   r   r   r   r    rt     s   rt   c                   @   s   e Zd Zdd Zdd ZdS )TestSparseRAdamc                 K   s   d| _ t|fddd|S )NTr"   )r%   ZenableRAdamrl   r)   r   r   r    r,     s    zTestSparseRAdam.build_optimizerc                 C   s   |  | j |  | j |  td td}tjj	t
dg|dd | jD ]}t| q^| jD ]}t| qxd S rb   rd   rf   r   r   r    r>     s    
zTestSparseRAdam.check_optimizerNrD   r   r   r   r    rv     s   rv   c                   @   s   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
eddd Zedeej ddd ZdS )TestYellowFinc                 C   s   d| _ t|ddS )NFr"   r%   )r(   r   )r   r*   r   r   r    r,   0  s    zTestYellowFin.build_optimizerc                 C   s   |  | j |  | j |  td td}tjj	t
dg|dd | jD ]}t| q^| jD ]}t| qxd S rb   rd   rf   r   r   r    r>   4  s    
zTestYellowFin.check_optimizerc                 C   s   t dd S r   r   r   r   r   r    r!   A  s    zTestYellowFin.testSparsec                 C   s   |r|d||   S |S d S )Nr.   r   )r   valrU   izero_debiasr   r   r    debD  s    zTestYellowFin.debc                 C   s"  |dkrN|| }t |d t |d  d }dt | d | }||fS |d |d  d | }t|| d| | |   | d }	|	dkrdndtt|	d	 }
|
|d
 |
  }|d }t|d}|| }tt |d t |d  d |d }dt | d | }||fS )Nr         gh/?ri   rS   r.   g      gUUUUUU?g      @g!?)r8   sqrtmathpowfabsminmax)r   ZdistanceZgrad_varh_minh_maxZdrmuZlr_minpZw3wyrootr   r   r    	get_lr_muJ  s     ("
*zTestYellowFin.get_lr_muc               
   C   sJ  i }d}d}d}	d}
d}t d}t d}t  t t tjJ |jg dd	gd
t j	j
d}|g dg}|||g|g W d    n1 s0    Y  |jg d|g|d}|r||gd}||gd}|j||gddd}|jg d|gdd}dd }||_||_tj|||	|
||d||| t| tj|dd t|D ]}t| td}|d	 }|d }|d }|d }td}tdd
 }tdd
 }| ||	|d	 |}t| ||	|d	 ||| |}|d
krPt| ||	|d	 |t| ||	|d	 ||| ||	|d	 |||d ||< qP|S )!Nr.   rS   +?   gư>netparam_init_net	iterationr}   r   )shapevalueZdtypeZiteration_mutexpre_grad)r   r   Ziteration_cpuiteration_floatgradT)	broadcastr   c                   S   s   d S rH   r   r   r   r   r    <lambda>  rr   z0TestYellowFin.caffe2_yellowfin.<locals>.<lambda>)rT   r   rU   curv_win_widthepsilonr{   )	overwriteZw_scalars_memoryr~         Zw_g_avgZw_lr_avgZw_mu_avgr   r   vardistlrr   )r   ZNetr   ZResetWorkspaceDeviceScopeDeviceOptionr   CPUZConstantFillrK   ZINT64ZCreateMutexZ
AtomicIterZCopyCPUToGPUZCastZMulZblobr   r<   ZYellowFinOptimizer_runZ
RunNetOnceZ	CreateNetrangeZRunNetr7   r|   r   dotr8   exp) r   r{   	grad_coefn_dimn_itergpuZ
caffe2_resrT   r   rU   r   r   r   r   r   Z
iter_mutexr   r   r   r   Z
param_inforz   Zscalars_memory_blobZg_norm2_avgZg_norm2_min_avgZg_norm2_max_avgZdistance_avgZ
g_avg_blobZres_lrZres_muZg_debZvariancer   r   r    caffe2_yellowfin]  s    

0






zTestYellowFin.caffe2_yellowfinc                 C   s  i }d}d}d}	d}
d}d}d}d}t |D ]L}|d | }d|	 dt|t|g d   }	d|
 dtj|t|g   }
d| d|  }d| dt|d |   }d| dttd|d d | d |   }|r"|	dd|d    |d | dd|d   d   }n|	|d |  }d| d|
 |	  }|dkr,|r| |dd|d    |t	|dd|d    t	|dd|d    \}}d| d|  }d| d|  }t	|dd|d    t	|dd|d    ||dd|d    ||d	||< q,| ||t	|t	|\}}d| d|  }d| d|  }t	|t	|||||d	||< q,|S )
NrS   r.   r}   r   gMbP?r~   r   r   r   )
r   r8   sumZonesZlinalgnormlogr   r   r   )r   r{   r   r   r   r   Z	numpy_resZtarget_h_maxZtarget_h_minZtarget_g_norm_squared_avgZtarget_g_norm_avgZtarget_g_avgZtarget_dist_avgZ	target_lrZ	target_murz   Zgrad_valZ
target_varr   r   r   r   r    numpy_yellowfin  s    &

	zTestYellowFin.numpy_yellowfinc                 C   s   ||||||}||||||}	t t|t|	 tdt|D ]}
t ||
  |	|
   ||
  D ]\}dt|
 d d | d d t| d d t| }t||
 | |	|
 | d|d qjq@d S )	Nr}   zi=z,
zfeat=z
grad_coef=zzero_debias={Gz?)Zrtolerr_msg)r   lenr   keysstrr   )r   Zmodel0Zmodel1r{   r   r   r   r   Z
model0_resZ
model1_resrz   Zfeatr   r   r   r    compare_yellowfin_models  s:    
z&TestYellowFin.compare_yellowfin_modelsz5Results might vary too much. Only for individual use.c                 C   sl   d}d}t tj}t |: dD ]$\}}| j| j| j||||dd q$W d    n1 s^0    Y  d S )N@B 2   ))Fr.   )Fr"   )Fr   )Tr.   Fr   )r   r   r   r   r   r   r   r   )r   r   r   Zcpu_device_optr{   r   r   r   r    test_caffe2_cpu_vs_numpy  s    z&TestYellowFin.test_caffe2_cpu_vs_numpyzNo gpu supportc                 C   st   d}d}t tjd}t |@ dD ]*}dD ] }| j| j| j||||dd q.q&W d    n1 sf0    Y  d S )Nr   r   r   )FT)r.   r"   r   Tr   )r   r   r   ZGpuDeviceTyper   r   r   r   )r   r   r   Zgpu_device_optr{   r   r   r   r    test_caffe2_gpu_vs_numpy*  s    z&TestYellowFin.test_caffe2_gpu_vs_numpyN)r@   rA   rB   r,   r>   r!   r|   r   r   r   r   r   skipr   rN   r   rO   r   r   r   r   r    rw   -  s   ZE
rw   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )TestRmsPropc                 K   s   d| _ t|fddd|S )NFr"   )r%   r   )r(   r   r)   r   r   r    r,   ?  s    zTestRmsProp.build_optimizerc                 C   s>   |  | j | | j | jD ]}t| q*d S rH   rV   rW   r   r   r    r>   E  s    zTestRmsProp.check_optimizerc                 C   s   t dd S r   r   r   r   r   r    r!   K  s    zTestRmsProp.testSparseNru   r   r   r   r    r   >  s   r   c                   @   s   e Zd Zdd ZdS )TestMultiOptimizersc                    s  ddl m}m}m  ddlm} |dd}||dddd	}|||d
d	d}|||ddd}||dgddg\}}	||	g  	| fdd}
 j
dd} j
dd}  }||
d" ||j|jdd W d    n1 s0    Y  ||
d" ||j|jdd W d    n1 s40    Y  g }|j jD ]B}|jdkrn|jd dks~|jd dkrN||jd  qN| |d |d  ||
d" ||j|jdd W d    n1 s0    Y  ||
d" ||j|jdd  W d    n1 s 0    Y  g }|j jD ]V}|jdkrZ|jd dksj|jd dkr:| |jd |v ||jd  q:| |d |d  ||
d!" ||j|jd!d" W d    n1 s0    Y  ||
d#" ||j|jd#d$ W d    n1 s 0    Y  g }|j jD ]j}|jd%krZ|jd d!ksj|jd d#kr:| |jd |v | |jd |v ||jd  q:| |d |d  d S )&Nr   )brewr   r<   ModelHelpertest)namedatafc1d   r   fc2   Zfc3
   labelsoftmaxlossc                    s     | d| S )Nz{}_grad)Zget_param_deviceformat)Z	blob_namer<   Zparam_to_devicer   r    infer_blob_device`  s    zGTestMultiOptimizers.test_multiple_optimizers.<locals>.infer_blob_devicer"   rx   皙?Zfc1_wZ
fc1_w_gradZfc1_bZ
fc1_b_gradWeightedSumr   r}   Zfc2_wZ
fc2_w_gradZfc2_bZ
fc2_b_gradZfc3_wZ
fc3_w_gradZfc3_bZ
fc3_b_gradZAdagrad)caffe2.pythonr   r   r<   caffe2.python.model_helperr   fcSoftmaxWithLossAddGradientOperatorsZ_get_param_to_devicer   ZAdagradOptimizerr   r   r   ProtooptypeinputappendassertEqualr2   )r   r   r   r   r*   r   r   predr   r   r   Zsgd_1Zsgd_2ZadagradZfc1_lr_blobsr   Zfc2_lr_blobsZfc3_lr_blobsr   r   r    test_multiple_optimizersP  sh    

022222z,TestMultiOptimizers.test_multiple_optimizersN)r@   rA   rB   r   r   r   r   r    r   O  s   r   c                   @   s   e Zd Zdd ZdS )TestWeightDecayc                 C   s(  ddl m} ddlm} |dddid}||dd	d
d
d}|||ddd}|||ddd}||dgddg\}}||g t|dd t	|d h d}	|j
 jD ]^}
|
jdkrd|
jv r|
jd |	vrtd|
jd  | |
jd |	v  |	|
jd  q| |	t d|	 d S )Nr   )r   r   r   orderNCHWr   Z	arg_scoper   cnv    r   ar      bra   r   r   r   -C6?Zweight_decay)\(?   Z
cnv_w_gradZb_w_gradZa_w_gradr   wd_0_0%Unexpected param for weight_decay: {} Not all weights were decayed: {})r   r   r   r   convr   r   r   r   r   r   r   r   r   r   outputprintr   r2   remover   set)r   r   r   r*   r   r   r   r   r   expected_weight_gradr   r   r   r    test_weight_decay  s:    
z!TestWeightDecay.test_weight_decayN)r@   rA   rB   r   r   r   r   r    r     s   r   c                   @   s   e Zd Zdd ZdS )TestOptimizerContextc              
   C   s*  ddl m}m} ddlm} |dddid}|jd }td	}td
}td}t|$ ||ddddd}	W d    n1 s0    Y  t||d" |	||	ddd}
W d    n1 s0    Y  |	||
ddd}|
|dgddg\}}||g t|dd t|d h d}d|dd|d d d|d! d"d|d# d$i}|j jD ]}|jd%krd&|jv r|jd |vrtd'|jd  | |jd |v  ||jd  |jd(krbd}|jD ]}|jd)kr|j}q| |||jd   qb| |t d*| d S )+Nr   )r   r<   r   r   r   r   r   r   g333333?r   r"   r   r   r   r   )ZWEIGHTZBIASr   r   r   r   ra   r   r   r   r   r   r   r   zSgdOptimizer_{}_lr_cpug333333ÿr}   gɿr~   gr   g)\(r   r   r   ZLearningRateZbase_lrr   )r   r   r<   r   r   Z_optimizer_instance_countr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r2   r   argr   fZassertAlmostEqualr   r   )r   r   r<   r   r*   countZ	cnv_optimZweight_optimZ
bias_optimr   r   r   r   r   r   Zexpected_learning_rater   ry   r   r   r   r    test_optimizer_context  sb    

20



z+TestOptimizerContext.test_optimizer_contextN)r@   rA   rB   r   r   r   r   r    r     s   r   )9Zcaffe2.protor   Zcaffe2.python.optimizerpythonr<   r   r   r   r   r   r   r	   r
   r   r   r   r   r   r   Zcaffe2.python.optimizer_contextr   Z!caffe2.python.optimizer_test_utilr   r   r   r   r   Zcaffe2.python.test_utilr   numpyr8   Znumpy.testingr   r   r   r   r   rC   rE   rG   rQ   rX   rY   r[   r`   rg   rh   rj   rk   rm   rt   rv   rw   r   r   r   r   r   r   r   r    <module>   sF   @
  B(