a
    ==icã#  ã                   @   sl   d dl Z d dlZd dlmZmZmZmZmZ d dl	m
Z
mZ d dlmZ G dd„ deƒZG dd„ deƒZdS )	é    N)ÚbrewÚcoreÚ	workspaceÚcnnÚ	optimizer)ÚInitializerÚPseudoFP16Initializer)ÚModelHelperc                   @   sP   e Zd ZdZejjfdd„Zdd„ Ze	 
ej d¡ejjfdd„ƒZd	d
„ ZdS )ÚOptimizerTestBaseú³
    This is an abstract base class.
    Don't inherit from unittest.TestCase, and don't name it 'Test*'.
    Do, however, do these things in classes which inherit from this.
    c                 C   s  t  g d¢¡ t j¡}t j d¡ |tjjkr4t jnt j	}|tjjkrJt
nt}t jjdd|jfd |¡}t  ||¡d d …t jf }tdddid	}tj|d
d|jddi fdi fd||d
}|tjjkrØ| ||d ¡}| |dg¡}	| |	d¡}
| |
g¡}|  |d tj¡ ||||fS )N©é   é   é   r   é   é{   r   é   ©ÚsizeÚtestÚorderÚNCHW)ÚnameZ	arg_scopeÚdataÚfcr   ÚConstantFillr   )ÚaxisZWeightInitializerZBiasInitializerZ_fp32ÚlabelÚavg_lossÚfc_w)ÚnpÚarrayÚastypeÚfloat32ÚrandomÚseedr   ÚDataTypeÚFLOATÚfloat16r   r   Úrandintr   ÚdotÚnewaxisr	   r   r   ÚFLOAT16ÚHalfToFloatÚSquaredL2DistanceÚAveragedLossÚAddGradientOperatorsÚassertIsInstanceZBlobReference)ÚselfÚdtypeÚperfect_modelZnumpy_dtypeZinitializerr   r   ÚmodelÚoutÚsqÚlossÚgrad_map© r:   úr/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/caffe2/python/optimizer_test_util.pyÚ_createDense   s6    ÿþþüzOptimizerTestBase._createDensec                 C   sÚ   |   ¡ \}}}}|  |¡}t d|d ¡ t d|d ¡ t |j¡ t |jd¡ tdƒD ]H}t	j
 |jd ¡}t d|| ¡ t d|| ¡ t |j ¡ j¡ q\t	jj|t	jd d …f t d¡dd |  |¡ d S )	Nr   r   r   TéÐ  r   ç{®Gáz„?©Zatol)r<   Úbuild_optimizerr   ÚFeedBlobÚ
RunNetOnceÚparam_init_netÚ	CreateNetÚnetÚranger    r$   r)   ÚshapeÚRunNetÚProtor   ÚtestingÚassert_allcloser+   Ú	FetchBlobÚcheck_optimizer)r2   r5   r4   r   r   r   Ú_Úidxr:   r:   r;   Ú	testDense2   s"    
ýzOptimizerTestBase.testDensezNo gpu supportc                 C   sö   t  tjd¡}t  |¡t |  |¡\}}}}|t jjkrR| dd¡}| 	|d¡ n| 	dd¡ t 
d|d ¡ t 
d|d ¡ W d   ƒ n1 s’0    Y  tj|dddd	dd
 |  |¡ | jrÆd S t |j¡ t |jd¡ t |j ¡ j¡ d S )Nr   r   Úfc_fp32_for_hostZfc_cpur   r   Zfc2r   é
   )Zdim_inZdim_outr   T)r   ZDeviceOptionr   ZGpuDeviceTypeZDeviceScoper<   r&   r,   r-   ZCopyGPUToCPUrA   r   r   r@   Z	_skip_gpurB   rC   rD   rE   rH   rI   r   )r2   r3   Z
device_optr5   Z_perfect_modelr   r   rQ   r:   r:   r;   ÚtestGPUDenseF   s     .
zOptimizerTestBase.testGPUDensec              	   C   sô  d}t  g d¢¡ t j¡}t j d¡ t jjdd|j| fd t j¡}t  |t  	||¡¡}t
jddd}|jjg d	|jgd
d}|j |¡ |j |dgd¡}| |d¡}| |dg¡}	| |	d¡}
| |
g¡}|  |d	 tj¡ |  |¡}t d¡ t d¡ t jt jfD ]Ö}t |j¡ t |jd¡ t dƒD ]”}t j |j!d ¡}t  	t  "|j¡|¡|| dk }|jdkrzq4t #d| $|jf¡ |¡¡ t #dt  || ¡ t j¡¡ t %|j &¡ j'¡ q4t j(j)|t *d	¡dd q|  +|¡ d S )Nr   r   r   r   r   r   r   )r   Úwç        )rG   ÚvalueÚindicesÚgatherÚsumr   r   Tr=   r   r   r>   r?   ),r    r!   r"   r#   r$   r%   r)   r   r*   Úrepeatr   ZCNNModelHelperrC   r   ÚparamsÚappendrE   ZGatherZReduceFrontSumr.   r/   r0   r1   r   ÚGradientSlicer@   r   Z
CreateBlobÚint32Úint64rB   rD   rF   rG   ZarangerA   ZreshaperH   rI   r   rJ   rK   rL   rM   )r2   ZDUPLICATIONr4   r   r   r5   rT   Zpickedr6   r7   r8   r9   r   Zindices_typerN   rO   rW   r:   r:   r;   Ú
testSparsea   sd    þþÿ


ÿ
ÿþÿý
zOptimizerTestBase.testSparseN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r&   r'   r<   rP   ÚunittestZskipIfr   Zhas_gpu_supportrS   r`   r:   r:   r:   r;   r
      s   r
   c                   @   s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	ÚLRModificationTestBaser   c                 C   s†   ddl m} d}|D ]N}t|j| |jƒs4|j| n
|j| j}t |¡}|t 	tj
 |¡d¡7 }qt |¡}	|}
|
t |
|	¡ }|S )Nr   )r   rU   g       @)Úcaffe2.pythonr   Ú
isinstanceÚparam_to_gradr]   Úvaluesr   rL   r    ÚpowerZlinalgÚnormÚsqrtÚmaximum)r2   r5   r[   Úmax_gradient_normr   Zsum_squared_normsÚparamZgradÚvalZglobal_normZ	clip_normÚ
norm_ratior:   r:   r;   Ú_gradient_ratio_referencež   s     þÿ
û

z0LRModificationTestBase._gradient_ratio_referencec                 C   s"  d}|   ¡ \}}}}| j||d}g }|jddD ]*}||jv r2t|j| tjƒs2| |¡ q2t 	d|d ¡ t 	d|d ¡ t 
|j¡ t |jd¡ |  |j¡ tj |jd ¡}	t 	d||	 ¡ t 	d||	 ¡ t |j ¡ j¡ |  |||¡}
t d¡}tj ||
¡ |  |
dk d	¡ d S )
Ng      ð?)ro   T)Z	top_scoper   r   r   z#norm_clipped_grad_update/norm_ratioz$Bad test, gradient not being scaled.)r<   r@   Z	GetParamsri   rh   r   r]   r\   r   rA   rB   rC   rD   rE   ÚassertIsNotNoneÚ_lr_multiplierr    r$   r)   rG   rH   rI   r   rs   rL   rJ   Zassert_almost_equalÚ
assertTrue)r2   ro   r5   r4   r   r   Úoptr[   rp   rO   Ú	referencerr   r:   r:   r;   Ú(test_global_norm_based_gradient_clipping°   s@    
þýÿÿz?LRModificationTestBase.test_global_norm_based_gradient_clippingc                 C   s¸   |   ¡ \}}}}| j|ddd}t d|d ¡ t d|d ¡ t |j¡ t |jd¡ |  |j	¡ |  
t ¡ d¡ t d¡ |  
t ¡ d¡ t |j ¡ j¡ |  
t d¡d¡ d S )Nr   T)ro   Zallow_lr_injectionr   r   r   Zlr_multiplier)r<   r@   r   rA   rB   rC   rD   rE   rt   ru   ÚassertEqualr   Zget_lr_injectionZset_lr_injectionrH   rI   r   rL   )r2   r5   r4   r   r   rw   r:   r:   r;   Útest_lr_injectionÖ   s    ÿ
z(LRModificationTestBase.test_lr_injectionN)ra   rb   rc   rd   rs   ry   r{   r:   r:   r:   r;   rf   —   s   &rf   )re   Únumpyr    rg   r   r   r   r   r   Z#caffe2.python.modeling.initializersr   r   Zcaffe2.python.model_helperr	   Úobjectr
   rf   r:   r:   r:   r;   Ú<module>   s    