a
    ==ic"                    @   s  d dl Zd dlZd dlZd dlmZmZ d dlmZm	Z	 d dl
mZmZmZmZ d dlmZ d dlZd dlZd dlmZmZmZmZ d dlm  mZ d dlmZ ed ej reZ!dd	 Zd
d Z"ej#dddZ$ddej%dfddZ&ej%ej'j(ej)ej'j*ej+ej'j,ej-ej'j.ej/ej'j0ej1ej'j2ej3ej'j4ej5ej'j6ej7ej'j8i	Z9dddZ:dej;dde:fddZ<dej;de:fddZ=G dd dej>Z?e@dkreA  dS )    N)partialreduce)	viewitemsviewkeys)assumegivensettingsHealthCheck)core	workspacett_coredyndep)
caffe2_pb2z*@/caffe2/caffe2/fb/optimizers:sgd_simd_opsc                     s,   d v rd  d<   dd  fdd}|S )Ndeadlinemax_examples2   c                    s   t f i  | S N)_hypothesis_settings)fkwargs n/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/caffe2/python/hypothesis_test.pywrapped   s    zsettings.<locals>.wrapped)
setdefault)r   r   r   r   r   r      s
    r   c                 C   s   ddt |    S )N      ?npexpxr   r   r   sigmoid!   s    r!         c                 K   s   | t jtjf i |||d}| t jtjf i |||d}tt|t| |k  | t|| ||| t|||fS )Nmin_sizemax_size)stlistshudimsr   lenarrays)Zdrawdtypeelementsmin_dimmax_dimr   Zdims_Zextra_r   r   r   _tensor_and_prefix%   s    r1   c                    sF   t jtjf i || |d}t jddd}t || fddS )zF generates a tensor and a list of indices of larger tensor of same dimr$      
   	min_value	max_valuec                    sN   t t | d t| d  tj| d d tjt jd| d d ddS )Nr"   r   r4   r-   r.   )r'   tuplesjustr)   r,   r   int64integers)pairr7   r   r   <lambda>5   s   z%_tensor_and_indices.<locals>.<lambda>)r'   r(   r)   r*   r;   r8   flatmap)r/   r0   r-   r.   r   Z
data_dims_Zoriginal_dimr   r7   r   _tensor_and_indices0   s    r?   c                 C   s"   | r| nt jt jt jg} t| S r   )r   int32r:   float32r'   sampled_fromdtypesr   r   r   _dtypesJ   s    rE   Fc                    sR   t f |  fddt|r"dndd|tdd dfdd	}|S )
Nc                    s   t jd| t j|  ddS )Nr2   filter_)nr-   r.   )r)   tensorselements_of_typer-   rF   r   r   r=   S   s   z_test_binary.<locals>.<lambda>)YX1X2)rL   )inputsout   r   r   c                    sr   t  ddg|g}|\}}| ||||gdg rZ|jtjkrZ| ||||gddg | ||||g d S )NrM   rN   r   )r
   CreateOperatorassertDeviceChecksr-   r   rA   assertGradientChecksassertReferenceChecks)selfrO   rP   gcdcoprM   rN   )namereftest_gradientr   r   test_binaryQ   s    	z!_test_binary.<locals>.test_binary)r   r>   r'   rB   r   )r[   r\   rG   gcsr]   allow_inplacerD   r^   r   )rG   r[   r\   r]   r   _test_binaryO   s    

	ra   c                    sT   t f |  fdd|r"t ntdd|tdddfdd	}|S )
Nc                    s   t | tj|  ddS )NrF   r7   )r1   r)   rJ   rK   rF   r   r   r=   h   s   z(_test_binary_broadcast.<locals>.<lambda>FrO   in_place   d   rR   c           	         sb   t j ddg|rdndgdd}|\}}| ||||gdg fdd}| ||||g| d S )	NrM   rN   rL   r"   )	broadcastr   c                    s   t j | |d | jdfS )Nr   rK   )r   arrayr-   r    y)r\   r   r   cast_reft   s    zG_test_binary_broadcast.<locals>.test_binary_broadcast.<locals>.cast_refr
   rS   rT   rV   )	rW   rO   rc   rX   rY   rZ   rM   rN   rj   )r[   r\   r   r   test_binary_broadcastg   s    z5_test_binary_broadcast.<locals>.test_binary_broadcast)r   r>   r'   booleansr9   r   )r[   r\   rG   r_   r`   rD   rl   r   )rG   r[   r\   r   _test_binary_broadcaste   s    
rn   c                   @   sX  e Zd Zdd Zef ejdde dej	e
dddd	 Zef d
ejddddieje
dddd Zef d
ejddieje
dddd Zdd Zdd Zdd Zdd Zef e e dej	e
dddd Zef de iej	e
dddd Zef de iej	e
dddd  Zef e e d!ej	e
ddd"d# Zeejdd$eejd%e d&e
ddd'd( Zef ejd)d$d*ejd$d+d*ed,d-gd.ej	e
ddd/d0 Zee j! d1e
d2deejd)d3d*ejd)d3d*e ed4ged5gej"d6d6d*ejdd7d*ejd)d$d*ejd)d$d*d8	d9d: Z#ef ed)d$ed;d3ed;d)edd$d<ej	e
d2d=d>d?d@ Z$ef edd$edAdBgdCej	e
dddDdE Z%ef dej&dFdgej"d6dGd*dHieje
dddIdJ Z'eee(j)e(j*e(j+e(j,gdKe
dddLdM Z-ef ejdde ej"dNdOd*e ej"dNdOd*dPej	e
dddQdR Z.ef ejd3de ej"dNdOd*ej"dNdOd*ej"dNdOd*ej"dSdTd*dUej	e
dddVdW Z/e0dXdY Z1ef ejd$de ej"dTdNd*ej"dNdOd*ej"dZdNd*ej"dZdNd*ed2d[gd\eje
ddd]d^ Z2e0d_d` Z3ef ejd$de ej"dTdNd*ej"dNdOd*ej"dZdNd*ej"dZdNd*ed2d[gd\eje
dddadb Z4ef ejd$dej"dTdNd*ej"dNdOd*ej"dZdNd*ej"dZdNd*ed2d[gdceje
ddddde Z5e0dfdg Z6ef ejd$de ej"dTdNd*ej"dNdOd*ej"dZdNd*ej"dZdNd*ed2d[gd\eje
dddhdi Z7ef ejd$dej"dTdNd*ej"dNdOd*ej"dZdNd*ej"dZdNd*ed2d[gdceje
dddjdk Z8ef ejdld)e(j+ejd;dmd*dne doej9e
dddpdq Z:ef ej&dmd3gej"drdrd;d)dsdHej&dmge(j+ejd;dd*dtejd)d3d*duej	e
dddvdw Z;ef dxej&dmgej"drdrdTd)dsdHiej	e
dddydz Z<ef d{ejejd;dmd*d;dmd|ieje
ddd}d~ Z=ef d{ejejd;dmd*d;dmd|ieje
dddd Z>ef i eje
dddd Z?ef d{ejejd;dmd*d;dmd|ieje
dddd Z@ef ejejd;dmd*d;dmd|e deje
d2ddd ZAef ej&dmd3gej"drdrd;d)dsdHej&dmge(j+ejd;dd*dtdej	e
dddd ZBef d{ejejd;dmd*d;dmd|ieje
dddd ZCef ejejd)dmd*d;dmd|eg ddeje
dddd ZDef dej&dmgej"drdrddHiej	e
dddd ZEef dej&dmgej"ddmd*dHiej	e
dddd ZFef dej&dmgej"ddmd*dHiej	e
dddd ZGef dej&dmgej"drdrddHiej	e
dddd ZHef dej&dmgej"d)dd*dHieje
dddd ZIdd ZJeed)dmed)ded)dFed)d3eejKde
dddd ZLeed)dmed)dmed)dFed)d3eejKde
d2d=d>dd ZMeed)dFedFdmed)dFed)d3de
d2d=d>dd ZNef de ieje
dddd ZOef i eje
dddd ZPef i ejdd ZQef i ejdd ZRef i ejdd ZSef i eje
dddd ZTef i ejdd ZUef i ejdd ZVef ed)dmeddge j!rdgng  dej	e
dddd ZWef ejdd7de e e e deje
d2d=d>ddĄ ZXef de ieje
ddddƄ ZYef de ieje
ddddȄ ZZef e e dɜeje
dddd˄ Z[eed;ded;dd̍e
dddd΄ Z\ddЄ Z]eed;ded;ddэe
ddddӄ Z^eed;ded;ded;dFdԍe
d2d=d>ddք Z_ef e ee`eaebee`eaebe dלej	e
ddddل Zcef e ej"ddTd*ejej"dTdd*d܍ej"ddZd*dݜej	e
dddd߄ Zdef ejej"ddd܍ej"ddmd*dej	e
dddd Zeef efe(j+e(jge(j)e(j,gdhdd e e ejd)dFed)dFd%dej	e
dddd Zief defe(j+e(jge(j)e(j,gdhdd iej	e
dddd Zjeed)dFed)dFed)dFde
dddd Zke
eljmgddef ed)dFed)dFed)dFed)dFed;dedd3dej	dd Zne
eljmgddef ed)dFed)dFed)dFed)dFed;dedd3dej	dd Zoef e e ej"ddd*dej	e
dddd Zpeeq ddd Zref ed;d3ed)dmed)dFed)dFedd geg ddej	e
d2d=d>dd Zsef ed;d3ed;d3ed;d3ed;d3ed)dmed)dFed)dFedd geg dd	ej	e
d2d=d>dd Ztef eddmed)dmed)d3edd gej"ddTd*d	eje
ddd
d Zuef def hdd iej	e
dddd Zvef d
ejddddieje
dddd Zwef ejddmd*ejddmd*ejddmd*ej"dNd6d*deje
dddd Zxef ejddmd*ejddmd*ej"dNd6d*deje
dddd Zyef ejddmd*ejddmd*deje
dddd Zzef ejdmdd*ejddmd*ejd)dFd*deje
dddd Z{e
d)d2deeddedFdmed)d)edddd d! Z|d2S ("  TestOperatorsc                 C   s\   dd dd dd dd d}t |D ]0\}}t||tjd|  t||tjd|  q&d S )Nc                 S   s
   | |k gS r   r   x1Zx2r   r   r   r=          z3TestOperators.test_comparison_ops.<locals>.<lambda>c                 S   s
   | |kgS r   r   rp   r   r   r   r=      rr   c                 S   s
   | |kgS r   r   rp   r   r   r   r=      rr   c                 S   s
   | |kgS r   r   rp   r   r   r   r=      rr   )LTZLEGTGE)r_   )r   ra   r)   gcs_cpu_onlyrn   )rW   opsr[   r\   r   r   r   test_comparison_ops   s    z!TestOperators.test_comparison_opsr2   rH   rb   i'  )r   c                 C   sV   t dddg|sdndg}|\}}| ||||gdg | ||||gddg d S )NSumrM   rN   rL   r   r
   rS   rT   rU   )rW   rO   rc   rX   rY   rZ   rM   rN   r   r   r   test_sum   s    zTestOperators.test_sumrO   )rH   r/   r0   c           
      C   s   t dddgdg}|\}}|d d df }dd }| ||||gdg tdD ]}	| ||||g|	dg qR| ||||g| d S )	NZRowMulrM   rN   rL   r   c                 S   s@   t j| j| jd}t|jD ]}| |f ||  ||f< q|gS )N)shaper-   )r   zerosr}   r-   rangesize)r    ri   retir   r   r   r\      s    z'TestOperators.test_row_mul.<locals>.refr2   )r
   rS   rT   r   rU   rV   )
rW   rO   rX   rY   rZ   rM   ZXtmprN   r\   r   r   r   r   test_row_mul   s    zTestOperators.test_row_mulc           	      C   s   t dddgdg}|\}}|t||d k||k  d8  < |t||d k||k  d7  < | ||||gdg tdD ]}| ||||g|dg qdd	 }| ||||g| d S )
NZMaxrM   rN   rL   g?r   r2   c                 S   s   t | |gS r   )r   maximumXrL   r   r   r   elementwise_max   s    z/TestOperators.test_max.<locals>.elementwise_max)r
   rS   r   logical_andrT   r   rU   rV   )	rW   rO   rX   rY   rZ   rM   rN   r   r   r   r   r   test_max   s    $$zTestOperators.test_maxc                 C   s:   dd }dd }t d||dd|  td||d|  d S )	Nc                 S   s   t | tst| dk S dS )Ni?T
isinstancefloatabsr   r   r   r   not_overflow   s    
z,TestOperators.test_add.<locals>.not_overflowc                 S   s
   | | fS r   r   rh   r   r   r   r\      s    z#TestOperators.test_add.<locals>.refAddTrG   r]   rF   ra   rn   rW   r   r\   r   r   r   test_add   s    zTestOperators.test_addc                 C   s,   dd }t d|dd|  td||  d S )Nc                 S   s
   | | fS r   r   rh   r   r   r   r\      s    z#TestOperators.test_sub.<locals>.refSubT)r]   r   )rW   r\   r   r   r   test_sub   s    zTestOperators.test_subc                 C   s:   dd }dd }t d||dd|  td||d|  d S )	Nc                 S   s   t | tst| dk S dS )Ni  Tr   r   r   r   r   r      s    
z,TestOperators.test_mul.<locals>.not_overflowc                 S   s
   | | fS r   r   rh   r   r   r   r\      s    z#TestOperators.test_mul.<locals>.refZMulTr   rF   r   r   r   r   r   test_mul   s    zTestOperators.test_mulc                 C   s`   dd }dd }dd }t d||d|tjd	|  t d||d
|d|  td|||d|  d S )Nc                 S   s
   | | fS r   r   rh   r   r   r   r\      s    z#TestOperators.test_div.<locals>.refc                 S   s   t | dkS )N{Gz?)r   r   r   r   r   non_zero   s    z(TestOperators.test_div.<locals>.non_zeroc                   S   s   t tjtjgS r   )r'   rB   r   rA   float64r   r   r   r   
div_dtypes   s    z*TestOperators.test_div.<locals>.div_dtypesDivT)rG   r]   rD   r_   F)rG   r]   rD   )rG   rD   )ra   r)   rv   rn   )rW   r\   r   r   r   r   r   test_div   s(    zTestOperators.test_div)r   rc     c                 C   sH   t ddg|sdndg}| |||gdg | |||gddg d S )NZNegativer   rL   r   r{   )rW   r   rc   rX   rY   rZ   r   r   r   test_negative   s
    
zTestOperators.test_negativer   c                 C   s<   t ddd}| |||gdg | |||gddg d S )NZTanhr   rL   r   r{   rW   r   rX   rY   rZ   r   r   r   	test_tanh   s    zTestOperators.test_tanhc                 C   s@   t ddgdg}| |||gdg | |||gddg d S )NAveragedLossr   lossr   r{   r   r   r   r   test_averaged_loss   s    z TestOperators.test_averaged_loss)r   inplacec                 C   s   t ddg|rdndg}dd }| |||gdg | |||g| |r| t& | |||gddg W d    q1 s0    Y  n| |||gddg d S )NZSoftsignr   rL   c                 S   s   | dt |   fS Nr"   r   r   )r   r   r   r   softsign  s    z-TestOperators.test_softsign.<locals>.softsignr   )r
   rS   rT   rV   assertRaises	ExceptionrU   )rW   r   r   rX   rY   rZ   r   r   r   r   test_softsign   s    6zTestOperators.test_softsignr#   )r%   r&   r.   )device_optionsset_seedc              	      s   t |}ttdd |D dk |r8|D ]
}d|_q,dd   fdd|D }|dd  D ]X}|rztj|d	 | q^| t	" tj|d	 | W d    q^1 s0    Y  q^d S )
Nc                 S   s   h | ]
}|j qS r   )device_type.0dor   r   r   	<setcomp>  rr   z;TestOperators.test_random_seed_behaviour.<locals>.<setcomp>r"   r   c                 S   sB   t   t j }tjdg dg| dgd}|| |jd  S )NZ
XavierFillrL   r2   )device_optionr}   )	r   ZResetWorkspaceC	Workspacer
   rS   runblobsfetch)r   wsrZ   r   r   r   r     s    

z5TestOperators.test_random_seed_behaviour.<locals>.runc                    s   g | ]} |qS r   r   r   r   r   r   
<listcomp>*  rr   z<TestOperators.test_random_seed_behaviour.<locals>.<listcomp>r   )
copydeepcopyr   r+   Zrandom_seedr   testingassert_array_equalr   AssertionError)rW   r   r   r   ysri   r   r   r   test_random_seed_behaviour  s    

z(TestOperators.test_random_seed_behaviourr"   r4       ZPACKED)axis
num_outputenginec                 C   sJ  t jd t jdddddt j}dd }|t|j|d  }|}	t j|	|t j}
t j|	t j}tj	dg ddg||d	}d
|fd|
fd|ffD ]\}}| j
|| q| j
| | j
jd  }| t|jt|jd | |	g  ||
|g}| |||dg t|D ]\}}| ||||dg q&d S )Nr"   r2   rd   c                 S   s   d}| D ]}||9 }q|S r   r   )xspr    r   r   r   prod;  s    
z5TestOperators.test_fully_connected_axis.<locals>.prodFC)r   WbrL   )r   r   r   r   r   r   )r   randomseedrandnastyperA   listr}   r
   rS   r   create_blobfeedr   r   r   assertEqualrT   	enumeraterU   )rW   r   r   r   rX   rY   r   r   KNr   r   rZ   r[   paramrL   rO   _r   r   r   test_fully_connected_axis2  s.    &
z'TestOperators.test_fully_connected_axisz$Skipping test due to no gpu present.Nrd   lstmZlinearr      )	hidden_size
num_layersbidirectionalrnn_mode
input_modedropoutTr   Dc
                 C   s  t jrt|dk d}
tj|
 t jr4tj}d}n
tj}d}||	 }|| }|rZ|d9 }|| }|}|}|rrdnd}|| | | }|| | | }d||d |   }||9 }tj	|
tj}| jdj||d tjd	g d
g d|||||||
|d}tj|||	
tj}| jdj||d | jjd  }tj|||| 
tj}|dkrtj|||| 
tjntd
tj}||||g}|dkrdd t|D ng d}|D ] }| j||||dgddd qd S )Nr"     ZMIOPENZCUDNNr2   r#   WEIGHTr   Z	Recurrent)INPUTZHIDDEN_INPUTZ
CELL_INPUTr   )ZOUTPUTZHIDDEN_OUTPUTZCELL_OUTPUTZRNN_SCRATCHZDROPOUT_STATES)r   r   r   r   r   r   r   r   r   r   r"   c                 S   s   g | ]\}}|qS r   r   )r   r   r   r   r   r   r     rr   z0TestOperators.test_recurrent.<locals>.<listcomp>)r   r"   rd   r   r   )stepsize	threshold)r   Zhas_hip_supportr   r   r   r   r)   Zhip_doZgpu_dorandr   rA   r   r   r   r
   rS   r   r   r   emptyr   rU   )rW   r   r   r   r   r   r   r   r   r   r   r   r   Zinput_weight_sizeZupper_layer_input_weight_sizeZrecurrent_weight_sizeZinput_bias_sizeZrecurrent_bias_sizeZnum_directionsZfirst_layer_szZupper_layer_szZtotal_szr   rZ   r   Hr   rO   Z
input_idxsZ	input_idxr   r   r   test_recurrentW  s    


zTestOperators.test_recurrentr   )ndimr   add_axis
num_inputsr   )r   r   c                    s   t |k  g dd | }g dd | }g dd | }	g }
t|D ]2} dkr`|	| |< |
tjj| tj qHtj	d|ddg d}| 
|||
dg t|D ]}| |||
|dg q fd	d
}| |||
| d S )NZX0rM   rN   ZX3r2   rd         )r"   r2   rd   r#   r   r   ConcatrL   Y_dims)r   r   c                     s\   t | }  r2tt| D ]}t| | | |< qtfdd| D }tj| d|gS )Nc                    s   g | ]}t |  qS r   r   r}   r   r    r   r   r   r     rr   zITestOperators.test_depth_concat.<locals>.depth_concat.<locals>.<listcomp>r   )r   r   r+   r   expand_dimsrg   concatenate)rO   r   
input_dimsr   r   r   r   depth_concat  s    z5TestOperators.test_depth_concat.<locals>.depth_concat)r   r   appendr   r   r   r   rA   r
   rS   rT   rU   rV   )rW   r   r   r   r   rX   rY   input_namesr}   individual_dimsrO   r   rZ   r   r   r   r   test_depth_concat  s"    zTestOperators.test_depth_concat)NCHWr"   )NHWCrd   )r   orderc                    s   g dd | }g d}g dd | }g }t |D ].}	||	 | d < |tjj| tj q4tjd|ddg d d	}
| 	||
|dg t |D ]}	| 
||
||	dg q fd
d}| ||
|| d S )Nr   r   )r"   r2   rd   r#   r"   r   rL   r   r   r  c                     s:   t | } d  t fdd| D }tj|  d|gS )Nr"   c                    s   g | ]}t |  qS r   r   r   r   r   r   r     rr   z_TestOperators.test_depth_concat_with_order.<locals>.depth_concat_with_order.<locals>.<listcomp>r   )r   r   rg   r   )rO   r   r  r   r   depth_concat_with_order  s    zKTestOperators.test_depth_concat_with_order.<locals>.depth_concat_with_order)r   r  r   r   r   r   rA   r
   rS   rT   rU   rV   )rW   r   r  rX   rY   r  r}   r  rO   r   rZ   r	  r   r  r   test_depth_concat_with_order  s    z*TestOperators.test_depth_concat_with_orderr   g      $@)r*   r.   c                 C   s   t d| t dtjdtjd t d td}|jg dddgdd	 t	d
}|
tjd
|gdd t | t d}t d}td|jd g}t|jd d D ]}	||	|jd   ||	d < qdd lm}
 |
j||dd d S )Ninputnextr   rK   outputcollect_net)r  r  r  r   )Znum_to_collectZcollect_datar2   num_iterr"   r   )decimal)r   FeedBlobr   rg   r@   Z
CreateBlobr
   NetZLastNWindowCollectorPlanAddStepexecution_stepZRunPlan	FetchBlobr~   r}   r   Znumpy.testingr   Zassert_almost_equal)rW   r   rX   rY   r  planr  rO   Z
new_outputr   Znptr   r   r   test_last_n_windows  s*    






z!TestOperators.test_last_n_windowsrK   c                 C   sB   t jd|}| jd| tddg }| j	| d S )Nr   dataZPrint)
r   r   Zpermutationr   r   r   r   r
   rS   r   )rW   r-   r  rZ   r   r   r   
test_print  s    zTestOperators.test_print皙??)rO   rc   momentumnesterovlrc                    s   |\}}	t j|gt jd}tjdg d|r.dnd|r8dndg t|d}
| ||
||	|gd	g  fd
d}| ||
||	|g| d S )NrK   ZMomentumSGD)gradmr   r!  grad_or"  Zm_o)r  r  r   r   c                    sP   |d }s$||   |  }||fS  | ||   }d  |  |  |fS d S )Nr   r"   r   )r!  r"  r   Zadjusted_gradientZm_newr  r  r   r   momentum_sgd  s    z5TestOperators.test_momentum_sgd.<locals>.momentum_sgd)r   asarrayrA   r
   rS   intrT   rV   )rW   rO   rc   r  r  r   rX   rY   r!  r"  rZ   r%  r   r$  r   test_momentum_sgd  s"    	

	zTestOperators.test_momentum_sgdgh㈵>r   )rO   rc   decayr  r   epsilonc	              	      s   |\}	}
}t |
d }
t j|gt jd}tjdg d|r>dnd|rHdnd|rRd	nd
g |d}| |||	|
||gdg  fdd}| |||	|
||g| d S )Nr   rK   ZRmsProp)r!  msmomr   r!  r#  r+  ms_or,  mom_o)r  r)  r*  r   r   c                    sN   |d }|d  t | |   }| ||  t |   }|}|||fS )Nr   r   )r   squaresqrt)r!  r+  r,  r   r-  r.  r#  r)  r*  r  r   r   rmsprop>  s
    z/TestOperators.test_rmsprop_sgd.<locals>.rmsprop)r   r   r&  rA   r
   rS   rT   rV   )rW   rO   rc   r)  r  r   r*  rX   rY   r!  r+  r,  rZ   r2  r   r1  r   test_rmsprop_sgd(  s    




zTestOperators.test_rmsprop_sgdc                 C   s   t | tjrt| } tj|ddd}tj|ddd}|| }	t||	 t| |  }
|||
|  7 }||	7 }t|| | |t| |  |  }d|t||k< |tj||gddfS )Nr   r   r"   )	r   r   ndarrayasscalartaker0  signr   stack)alphabetalambda1lambda2wnzgrH   zg2sigmar   r   r   _dense_ftrlG  s    
zTestOperators._dense_ftrlgMbP?ZSIMD)rO   rc   r:  r;  r<  r=  r   c
                 C   s   |\}
}}}t |}t j||gdd}tjdg d|r<dnd|rFdndg||||||d		}| |	||
||gd
g | |||
||gt| j|||| d S )Nr4  r   Ftrlvarr?  r!  rG  var_or?  nz_or:  r;  r<  r=  r   r   r   )	r   r   r9  r
   rS   rT   rV   r   rD  rW   rO   rc   r:  r;  r<  r=  r   rX   rY   rG  rH   rA  r!  r?  rZ   r   r   r   test_ftrl_sgdW  s(    


zTestOperators.test_ftrl_sgdc                 C   sr  t | tjrt| } |j}tj|ddd}tj|ddd}	|jd }
||
d}||
d}||
d}|	|
d}	|jd }|| }t|| t| |  }|	|||  7 }	||7 }tjj	|	ddd}|d }|	|t|
 | d  |t| |  |  }t
|D ].}|| |t|
 krd|d d |f< q||}||}|	|}	|tj||	gddfS )Nr   r4  r   r"   r2   gư>)r   r   r5  r6  r}   r7  reshaper0  linalgnormr   r9  )r:  r;  r<  r=  r>  r?  r@  Z	old_shaperH   rA  Z
output_dimZ	input_dimrB  rC  Zz_normsr   r   r   r   _dense_gftrlu  s6    





zTestOperators._dense_gftrlc
                 C   s   |\}
}}}t |}t j||gdd}tjdg d|r<dnd|rFdndg||||||d		}| |	||
||gd
g | |||
||gt| j|||| d S )Nr4  r   ZGFtrlrF  rG  rH  r?  rI  rJ  r   )	r   r   r9  r
   rS   rT   rV   r   rP  rK  r   r   r   test_gftrl_sgd  s(    


zTestOperators.test_gftrl_sgd)rO   r:  r;  r<  r=  r   c	                    s   |\}	}
}}t |	jd }||d dk }|| }t |
}
t j|
|gdd}tjdg dddg ||d		}|||	|||gdg  fd
d}|||	|||g| d S )Nr   r2   r4  r   
SparseFtrl)rG  r?  indicesr!  rG  r?  rJ  c              	      s:     | | || |\}}|| |< |||< | |fS r   )rD  )r>  r?  r   r@  swsnzr:  r;  r<  r=  rW   r   r   ftrl  s    z0TestOperators.test_sparse_ftrl_sgd.<locals>.ftrl)	r   aranger}   r   r9  r
   rS   rT   rV   rW   rO   r:  r;  r<  r=  r   rX   rY   rG  rH   rA  r!  rS  r?  rZ   rW  r   rV  r   test_sparse_ftrl_sgd  s&    

z"TestOperators.test_sparse_ftrl_sgdc              	   C   s   t || |||||S r   )ro   rD  )r;  r<  r=  r>  r?  r@  r:  r   r   r   _dense_ftrl_send_alpha_by_input  s    z-TestOperators._dense_ftrl_send_alpha_by_inputc
              
   C   s   |\}
}}}t |}t j||gdd}t |t j}tjdg d|rNdnd|rXdndg|||||d	}| |	||
|||gd
g | 	|||
|||gt
| j||| d S )Nr4  r   rE  )rG  r?  r!  r:  rG  rH  r?  rI  r;  r<  r=  r   r   r   )r   r   r9  rg   r   rA   r
   rS   rT   rV   r   r[  rK  r   r   r   !test_ftrl_sgd_send_alpha_by_input  s*    


z/TestOperators.test_ftrl_sgd_send_alpha_by_inputc	              
      s   |\}	}
}}t |	jd }||d dk }|| }t |
}
t j|
|gdd}t |t j}tj	dg dddg ||d	}
|||	||||gdg  fd
d}|||	||||g| d S )Nr   r2   r4  r   rR  )rG  r?  rS  r!  r:  rG  r?  r\  c              	      s:     | | || ||\}}|| |< |||< | |fS r   )r[  )r>  r?  r   r@  r:  rT  rU  r;  r<  r=  rW   r   r   rW     s    zDTestOperators.test_sparse_ftrl_sgd_send_alpha_by_input.<locals>.ftrl)r   rX  r}   r   r9  rg   r   rA   r
   rS   rT   rV   rY  r   r^  r   (test_sparse_ftrl_sgd_send_alpha_by_input  s,    
z6TestOperators.test_sparse_ftrl_sgd_send_alpha_by_inputrQ   r3   )r6   r0   r-   r.   )r  with_remappingc                    s^   t jddgdgrdgng  |d} |||gdg d	 fdd	} |||g| d S )
NZUniquer  unique	remappingr   r   c                    sp    jtt|   ttt|  rl  j| j  fddtt| D }tj	||  d S )Nc                    s   g | ]} |  qS r   r   r   r   rb  ra  r   r   r   @  rr   zCTestOperators.test_unique.<locals>.unique_valid.<locals>.<listcomp>)
r   r   r+   setsortedr}   r   r   r   r   )r  ra  rb  ZremappedrW   r`  rd  r   unique_valid;  s    z/TestOperators.test_unique.<locals>.unique_valid)N)r
   rS   rT   ZassertValidationChecks)rW   r  r`  rX   rY   rZ   rh  r   rg  r   test_unique+  s    zTestOperators.test_uniqueF)	allow_nanallow_infinityr5   r6   )r*   r-   r.   )
predictionlabelstop_kc                 C   sL   |dkrt j}tjdddgdg||d}dd }| j|||||g|d	 d S )
Nr"   ZAccuracyrl  rm  accuracy)rn  r   c           
      S   s   | j d }d}tdt| D ]\}tdd t| | D dd dd}dd	 |d| D }|D ]}||| kr^|d
7 }q^q|| }	|	fS )Nr   c                 s   s   | ]\}}||gV  qd S r   r   )r   jitemr   r   r   	<genexpr>b  rr   z>TestOperators.test_accuracy.<locals>.op_ref.<locals>.<genexpr>c                 S   s   | d S Nr   r   r   r   r   r   r=   c  rr   z=TestOperators.test_accuracy.<locals>.op_ref.<locals>.<lambda>T)keyreversec                 S   s   g | ]}|d  qS r   r   r   r   r   r   r   f  rr   z?TestOperators.test_accuracy.<locals>.op_ref.<locals>.<listcomp>r"   )r}   r   r+   rf  r   )
rl  rm  rn  r   Zcorrectr   Zpred_sortedmax_idsr"  ro  r   r   r   op_ref]  s    
z+TestOperators.test_accuracy.<locals>.op_refr   rZ   rO   	reference)r)   cpu_dor
   rS   rV   )rW   rl  rm  rn  rX   rY   rZ   rw  r   r   r   test_accuracyE  s     zTestOperators.test_accuracytarget_probabilitiesc                 C   s2   t ddgdg}dd }| j|||g|d d S )NZ
Perplexityr|  
perplexityc                 S   s.   | j d }t| d| }tdd |}|fS )Nr   g      c                 S   s   | | S r   r   rh   r   r   r   r=     rr   z?TestOperators.test_perplexity.<locals>.op_ref.<locals>.<lambda>)r}   r   powerr   )r|  r   Zperplexitiesr}  r   r   r   rw    s    
z-TestOperators.test_perplexity.<locals>.op_refrx  r
   rS   rV   )rW   r|  rX   rY   rZ   rw  r   r   r   test_perplexitys  s    zTestOperators.test_perplexitylengthsr$   c                 C   s>   t ddgdg}dd }| j||tj|tjdg|d d S )NZLengthsToSegmentIdsr  segment_idsc                 S   s8   g }t | D ]\}}|||g  qtj|tjdfS NrK   r   extendr   rg   r@   r  sidsr   lr   r   r   rw    s    z9TestOperators.test_lengths_to_segment_ids.<locals>.op_refrK   rx  r
   rS   rV   r   rg   r@   rW   r  rX   rY   rZ   rw  r   r   r   test_lengths_to_segment_ids  s    z)TestOperators.test_lengths_to_segment_idsc                 C   s>   t ddgdg}dd }| j||tj|tjdg|d d S )NZLengthsRangeFillr  Zincreasing_seqc                 S   s:   g }t | D ]\}}|tt| qtj|tjdfS r  )r   r  r   r   r   rg   r@   )r  r  r   r  r   r   r   rw    s    z5TestOperators.test_lengths_range_fill.<locals>.op_refrK   rx  r  r  r   r   r   test_lengths_range_fill  s    z%TestOperators.test_lengths_range_fillc                 C   sP   g d}t ddgdg}dd }dd }| j||tj||tjd	|d
 d S )N)r#   r   rd   r2   r   r#   ZSegmentIdsToRangesr  rangesc                 S   s~   t jddgt jdg}d}t| D ]H\}}||krV|d7 }|t j|dgt jd q*|d d  d7  < q"t j|t jdfS )Nr   rK   r"   r4  )r   rg   r@   r   r  )r  r  prevr   Zsidr   r   r   rw    s    z8TestOperators.test_segment_ids_to_ranges.<locals>.op_refc                 S   s8   g }t | D ]\}}|||g  qtj|tjdfS r  r  r  r   r   r   lengths_to_segment_ids  s    zHTestOperators.test_segment_ids_to_ranges.<locals>.lengths_to_segment_idsrK   rx  r  )rW   rX   rY   r  rZ   rw  r  r   r   r   test_segment_ids_to_ranges  s    
z(TestOperators.test_segment_ids_to_rangesc                 C   s>   t ddgdg}dd }| j||tj|tjdg|d d S )NZLengthsToRangesr  r  c                 S   s<   | j s| dfS ttdgt| d d f| ffS )N)r   r2   r   r4  )r   rM  r   Zcolumn_stackr   cumsumr   r   r   r   rw    s
     z4TestOperators.test_lengths_to_ranges.<locals>.op_refrK   rx  r  r  r   r   r   test_lengths_to_ranges  s    z$TestOperators.test_lengths_to_ranges)r  include_last_offsetc                    sF   t jddgdg d} fdd}| j||tj|tjdg|d d S )	NZLengthsToOffsetsr  r  r  c                    sd   | j s| dg}n tdgt| d d fg} r\t|d tt| gf|d< t|S )Nr   r4  )r   rM  r   r   r  rg   sumtuple)r    arrr  r   r   rw    s     $z5TestOperators.test_lengths_to_offsets.<locals>.op_refrK   rx  r  )rW   r  r  rX   rY   rZ   rw  r   r  r   test_lengths_to_offsets  s    		z%TestOperators.test_lengths_to_offsets)rl  rm  c                 C   s8   t dddgddg}dd }| j||||g|d d S )	NZMultiClassAccuracyrl  rm  
accuraciesamountsc                 S   s   | j d }| j d }tj|td}|d tj|td}|d tj| dd}td|D ]<}|| }|| }	||	kr||	  d7  < ||	  d7  < q\td|D ] }|| }
|
r||  |
  < q||fS )Nr   r"   rK   r   )r}   r   r   r   fillr'  Zargmaxr   )rl  rm  r   r   r  r  rv  r   Zmax_idZlabel_idamountr   r   r   rw  #  s$    



z7TestOperators.test_multi_class_accuracy.<locals>.op_refrx  r  )rW   rl  rm  rX   rY   rZ   rw  r   r   r   test_multi_class_accuracy  s    z'TestOperators.test_multi_class_accuracyc                 C   sN   t ddgdg}dd }||}dd }| j||tj|tjdg|d	 d S )
NZSegmentIdsToLengthsr  r  c                 S   s*   g }t | D ]\}}|||g  q|S r   )r   r  r  r   r   r   lengths_to_idsH  s    zATestOperators.test_segment_ids_to_lengths.<locals>.lengths_to_idsc                 S   s   t | }|dkr"tjg tjdfS g }d}d}t|D ]V}| | }||kr|dkr\|| |d |kr||d |d7 }q\|}d}|d7 }q6|| tj|tjdfS )Nr   rK   r4  r"   )r+   r   rg   r@   r   r  )ZidsZ
ids_lengthr  Zprev_idZ
tmp_lengthidxZcur_idr   r   r   ids_to_lengthsP  s&    




zATestOperators.test_segment_ids_to_lengths.<locals>.ids_to_lengthsrK   rx  r  )rW   r  rX   rY   rZ   r  r  r  r   r   r   test_segment_ids_to_lengths=  s    z)TestOperators.test_segment_ids_to_lengths)      ?r   g      ?       @)r  r~  c                    sF   t jddgdg d} fdd}| j||tj|tjdg|d d S )	NZLengthsToWeightsr  weightsr~  c                    s8   g }| D ]}| |dt|  g  qtj|tdfS Nr"   rK   )r  powr   rg   r   )r  Zweighted_lengthr  r  r   r   lengths_to_weightsz  s    zATestOperators.test_lengths_to_weights.<locals>.lengths_to_weightsrK   rx  r  )rW   r  r~  rX   rY   rZ   r  r   r  r   test_lengths_to_weightsm  s    z%TestOperators.test_lengths_to_weightsinput_tensor)rj  rk  c                 C   s2   t ddgdg}dd }| j|||g|d d S )NZAbsr  r  c                 S   s   t | fS r   r   r  r   r   r   abs_ref  s    z'TestOperators.test_abs.<locals>.abs_refrx  r  )rW   r  rX   rY   rZ   r  r   r   r   test_abs  s    zTestOperators.test_absic                 C   s2   t ddgdg}dd }| j|||g|d d S )NZCosr  r  c                 S   s   t | fS r   )r   cosr  r   r   r   cos_ref  s    z'TestOperators.test_cos.<locals>.cos_refrx  r  )rW   r  rX   rY   rZ   r  r   r   r   test_cos  s    zTestOperators.test_cosc                 C   s2   t ddgdg}dd }| j|||g|d d S )NZSinr  r  c                 S   s   t | fS r   )r   sinr  r   r   r   sin_ref  s    z'TestOperators.test_sin.<locals>.sin_refrx  r  )rW   r  rX   rY   rZ   r  r   r   r   test_sin  s    zTestOperators.test_sinc                 C   s2   t ddgdg}dd }| j|||g|d d S )NZExpr  r  c                 S   s   t | fS r   r   r  r   r   r   exp_ref  s    z'TestOperators.test_exp.<locals>.exp_refrx  r  )rW   r  rX   rY   rZ   r  r   r   r   test_exp  s    zTestOperators.test_expc                 C   sH   t ddgdg}dd }| j|||g|d | |||gddg d S )NLogr  r  c                 S   s   t | fS r   )r   logr  r   r   r   log_ref  s    z'TestOperators.test_log.<locals>.log_refrx  r   )r
   rS   rV   rU   )rW   r  rX   rY   rZ   r  r   r   r   test_log  s    zTestOperators.test_logc                    sr   t jdg dgddd j  t }t jddgdgdd	 t fd
d t | }|d d S )NCreateBlobsQueuequeuer   r"   capacity	num_blobsDequeueBlobsrP   g?)Ztimeout_secsc                      s   j  S r   )r   r   r   rZ   rW   r   r   r=     rr   z:TestOperators.test_blobs_dequeue_timeout.<locals>.<lambda>gRQ?)r
   rS   r   r   timer   RuntimeErrorZassertGreater)rW   tr   r  r   test_blobs_dequeue_timeout  s$    z(TestOperators.test_blobs_dequeue_timeoutre   )num_threadsnum_elementsr  r  r   c              	      s  ddl 
ddltjdg dg| d}	j| fddtD  tD ]fddD  qZ 	fd	d
t|D ]&}tD ]	j	d
| qq
fddt|D }|D ]}	|	  qtD ]:fddtD }
tjddg|
 d}	j| q|D ]}	|	  q.tddgg }	j| 	fddtD tD ]T	 j j tD ],	tfddtD  qqzdS )a  
        - Construct matrices of size N x D
        - Start K threads
        - Push all N rows into the queue of capacity C
        - Pull all N rows out of the queue.
        - Verify that the output matrices are permutation of the rows of the
          original matrices.
        r   Nr  r  )r  r  r   c                    s"   g | ]}t j d t jqS )r   )r   r   r   r   rA   r   r   )r  r   r   r     s   z<TestOperators.test_blobs_queue_threading.<locals>.<listcomp>c                    s   g | ]}|  qS r   r   r   r   r   r   r   "  rr   c                    s    fddt D }tjddg| |d}zB }t||D ]\}}j|j|d qBj| W q  j	y   Y d S 0 q d S )Nc                    s   g | ]}d  | qS )x_{}_{}formatrc  r  r   r   r   &  rr   zMTestOperators.test_blobs_queue_threading.<locals>.enqueue.<locals>.<listcomp>ZEnqueueBlobsr  r   )
r   r
   rS   
get_nowaitzipr   r   r   r   Empty)r  Z
feed_blobsrZ   elemselemZ	feed_blob)r   r  qr  rW   r  r   enqueue$  s     z9TestOperators.test_blobs_queue_threading.<locals>.enqueuer  c                    s   g | ]}j  |fd qS )targetargs)Thread)r   r  )r  	threadingr   r   r   ;  s   c                    s   g | ]}d  | qS zy_{}_{}r  rc  ry   r   r   r   A  rr   r  r   CloseBlobsQueuec                    s*   g | ]" t  fd dtD qS )c                    s$   g | ]}j jd  |  qS r  )r   r   r  r   )r   rH   )r   rW   r   r   r   L  s   zGTestOperators.test_blobs_queue_threading.<locals>.<listcomp>.<listcomp>)r   Zvstackr   )r   )r  rW   r  r   r   L  s   c                 3   s*   | ]"}t      | V  qd S r   )r   Zarray_equal)r   k)r   rp  r   r   r   r   rr  V  s   z;TestOperators.test_blobs_queue_threading.<locals>.<genexpr>)r  r  r
   rS   r   r   r   Queueputr   r  startjoinr   r}   
assertTrueany)rW   r  r  r  r  r   rZ   r  threadsthreadZdequeue_blobsr   )r   r  r   rp  rH   r  r  r  r  rW   r  r   r   r   test_blobs_queue_threading  sd    

z(TestOperators.test_blobs_queue_threading)num_producersnum_consumersr  r  r   c                    s   t d}|jg d||d}g }d}	t|D ]z}
d|
 }t |  fddt|D }  } |g| ||g  |
d d }t j| |d	}|	|7 }	|| q,t d
}||gd t dt jd|ddt d|g}g }g }|j	g ddd}t|D ]}
d|
 }t |}|
|g|d }|d }t |d }|j	g ddd}|| |||g| |t j|||g|d qt jd|dd}t d|}t jd||gdd}t d}|| || | j| d}|D ]"}|| jjt|   7 }q| ||	 d S )Ninit_netr"   r  r   producer_%dc                    s   g | ]} j g d dddqS r"   r   F)valueZrun_onceConstantFill)r   timesnetr   r   r   i  s   z7TestOperators.test_safe_blobs_queue.<locals>.<listcomp>r3   r  producer_exit_netproducer	producersTZconcurrent_substepsproducer_exitr   r  zconsumer_%dr4  _counter        Zshould_stop_blobconsumerinitworkertest)r
   r  r  r   NextNameSafeEnqueueBlobsr  r  r  r  SafeDequeueBlobsr   r  r  r   r   r   strr   tolistr   )rW   r  r  r  r  r   r  r  producer_stepstruthr   r[   r   statuscountstepr  producer_stepZconsumer_stepsZcountersZconst_1Znet1Znet2counterconsumer_step	init_stepworker_stepr  vr   r  r   test_safe_blobs_queueY  sp    











 z#TestOperators.test_safe_blobs_queue)
num_queuesr  r  r  c              	      s  t d| td fddt|D }g }g }t|D ]}d| }	t|	fddtD }
 }|| g|
 |
|g  td| }||| d || t|	tjd| g|d	td| |gg}|| q>td
tjd|ddg}g fdd}td}t	j
jddd}|j|d t	j
jdd|fd|d d}
|
d }||| tdtjd|g|dtd|g}td}tjd||gdd}td}|| || | j| t|d ksJ t|| d ksJ d S )Nr  r  c                    s   g | ]}j g d  dqS )r"   r  )r  r  )r  r  r  r   r   r     s   zBTestOperators.test_weighted_sample_blobs_queue.<locals>.<listcomp>r  c                    s   g | ]} j g d dddqS r  r  r  r  r   r   r     s   zproducer_exit_%dr   r  r  r  Tr  c                    s     |  d S r   )r  )insZouts)
status_lstr   r   r    s    z>TestOperators.test_weighted_sample_blobs_queue.<locals>.appendZweight_sample_dequeue_netr4  r"   )lowhighr   r  r   )r  table_idx_blobr  r  r  r  r  r  )printr
   r  r   r  r  r  r  r  r   r   randintZWeightedSampleDequeueBlobsuniformPythonr  r  r   r   r+   )rW   r  r  r  r  queuesr  Zproducer_exit_netsr   r[   r   r  Zexit_netr  r  r  Zconsumer_netr  r	  r
  r  r  r   )r  r  r  r  r  r    test_weighted_sample_blobs_queue  s    	












z.TestOperators.test_weighted_sample_blobs_queuer  c                    sj   ddg t |jdkr  d tjddgdg d} fdd} fd	d
}| j|||g|d|d d S )Nr   r2   Z
ExpandDimsr  expandedr*   c                    s6   t t }|  | }|D ]}tj||d}q|fS )Nr   )r   re  sortr   r   )r  r  kwZinc_dimsrdimr  r   r   expand_dims_ref  s    z?TestOperators.test_squeeze_expand_dims.<locals>.expand_dims_refc                    s:   t t }|jdd | }|D ]}tj||d}q |fS )NT)ru  r   )r   re  r  r   Zsqueeze)r  r  r  Zdec_dimsr  r  r  r   r   squeeze_ref
  s    z;TestOperators.test_squeeze_expand_dims.<locals>.squeeze_ref)r   rZ   rO   ry  Zoutput_to_gradZgrad_reference)r+   r}   r  r
   rS   rV   )rW   r  rX   rY   rZ   r   r!  r   r  r   test_squeeze_expand_dims  s&    
z&TestOperators.test_squeeze_expand_dimsc                 C   s  d}t j| g d}g d}g d}tjdg ddg|||d}t jt jdt jd	d
}t 	d	gd t j}	t
|||}
| jd| | jd|	 | jd|
 | j| | jjd  }|dg}t 	g d}| jt j|| d	dd d S )Nr   )r2   r2   r2   r2   )r"   rd   rd   rd   r"   ZTT)r   r   coresrL   )	inp_sizes	out_sizestt_ranks   r   r   r   r   r#  )g=]ﯾgލg<E6gå~÷>gtg)|
g&
>>g^jgU->gP>gFSḴgwrD{gcPrWq>g⩰KgLg~g|=delta)r   r   r   r
   rS   r   r   r   rA   rg   r   Zinit_tt_coresr   r   r   r   r   r   rM  assertAlmostEqualrN  rO  )rW   rX   rY   r   r$  r%  r&  rZ   r   r   r#  rL   goldenr   r   r   test_tt_layer  s4    	zTestOperators.test_tt_layerc              	   C   sl  d}t j| g d}g d}tjdg dg d||g ddd	}t g d
t j}t g dt j}t g dt j}	t g dt j	}
t ddgt j
}| jd| | jd| | jd|	 | jd|
 | jd| | j| | jjd  }| t|jddg t g dg dg}| jt j|| ddd d S )Nr   r3   r3   r3   r2   r2   r2   TTSparseLengthsSumcore0core1core2indexr  rL   Zcore0_outputZcore1_outputrS  r"   r'  r'  r"   r   Zfactor_iZfactor_jZranksZemb_sizer3   r"   r2   r'  r3   r'  r2   r'  r3   r'  r2   r"   r   r"   r2   r"   r#   rd   r2   r1  r2  r3  r4  r  rL   )   r<  r<  r<  r<  r<  r<  r<  )   r=  r=  r=  r=  r=  r=  r=  r   r(  )r   r   r   r
   rS   onesr   rA   rg   r:   r@   r   r   r   r   r   r   r   r   r}   r*  rN  rO  )rW   rX   rY   r   
factor_vocfactor_widthrZ   c0c1c2r4  r  rL   r+  r   r   r   test_tt_sls_layerG  s<    	
zTestOperators.test_tt_sls_layerc                 C   s  t dg dg d}tg dtj}tg dtj}tg dtj}tddgtj}tg d	tj}tg d
tj}	tg dg dg dg dg dgtj}
tddgtj}| j	
d| | j	
d| | j	
d| | j	
d| | j	
d| | j	
d|	 | j	
d|
 | j	
d| | j	| | j	jd  }| j	jd  }| j	jd  }| t|jt|j | t|jt|j | t|jt|j d S )NTTSparseLengthsSumGradientr1  r2  r3  r  	core0_out	core1_outrS  dYdCore0dCore1dCore2)r3   r"   r#   r'  )r3   r'  r#   r'  )r3   r'  r#   r"   rd   r2   r   r#   r'  )r   r'  r'  r   r   r   r"   r   r   r2   r   r   r#   r   r   @   r1  r2  r3  r  rG  rH  rS  rI  rK  rL  rM  r
   rS   r   r>  r   rA   rg   r@   r:   r   r   r   r   r   r   r   r   r}   rW   rX   rY   rZ   rA  rB  rC  r  Zc0_outZc1_outrS  rI  rK  rL  rM  r   r   r   test_tt_sls_gradientopo  sF    
z$TestOperators.test_tt_sls_gradientopc                 C   s  t dg dg d}tg dtj}tg dtj}tg dtj}tddgtj}tg d	tj}tg d
tj}	tg dg dg dg dg dgtj}
tddgtj}| j	
d| | j	
d| | j	
d| | j	
d| | j	
d| | j	
d|	 | j	
d|
 | j	
d| | j	| | j	jd  }| j	jd  }| j	jd  }| t|jt|j | t|jt|j | t|jt|j d S )NrE  rF  rJ  e   r"   r2   r'  f   r'  r2   r'     r'  r#   r"   rd   r2   r   r2   r'  rN  rO  rP  rQ  rR  r'  r1  r2  r3  r  rG  rH  rS  rI  rK  rL  rM  rT  rU  r   r   r   test_tt_sls_gradientop1  sF    
z%TestOperators.test_tt_sls_gradientop1c              	   C   s   g d}g d}t jdg dg d||g ddd}tg d	tj}tg d
tj}tg dtj}tg dtj}	tg dtj}
| 	||||||	|
gddg d S )Nr-  r.  r/  r0  r5  r6  r   r7  r8  r9  r:  r;  r   rd   r   r   r2   r   r   r   )
r
   rS   r   r>  r   rA   rg   r:   r@   rU   )rW   rX   rY   r?  r@  rZ   rA  rB  rC  r4  r  r   r   r   test_tt_sls  s"    	zTestOperators.test_tt_slsc              
   C   s  g d}g d}t jdg dg d||g ddd}tg d	tj}tg d
tj}tg dtj}tg dtj}	tg dtj}
| j	
d| | j	
d| | j	
d| | j	
d|	 | j	
d|
 | j	| | j	jd  }| t|jddg tdgd dgd dgd dgd dgd dgd dgd g}| jtj|| ddd d S )N)}         )r#   r#   r#   r/  r0  r5  r6  rS  r7  )ra  r"   r#   r'  )rb  r'  r#   r'  )rc  r'  r#   r"   )r   = i N  i@B rd  r_  r1  r2  r3  r4  r  rL   r   r   r<  r=  r(  )r
   rS   r   r>  r   rA   rg   r:   r@   r   r   r   r   r   r   r   r   r}   r*  rN  rO  )rW   rX   rY   r?  r@  rZ   rA  rB  rC  r4  r  rL   r+  r   r   r   test_tt_sls_repro  s4    	BzTestOperators.test_tt_sls_reproc                 C   s  t dg dg d}tg dtj}tg dtj}tg dtj}tg dtj}tg dtj}tg d	tj}	tg d
g dg dg dg dgtj}
tddgtj}| j	
d| | j	
d| | j	
d| | j	
d| | j	
d| | j	
d|	 | j	
d|
 | j	
d| | j	| | j	jd  }| j	jd  }| j	jd  }| t|jt|j | t|jt|j | t|jt|j d S )NrE  rF  rJ  rW  rY  r[  r_  r]  rN  rO  rP  rQ  rR  r   r'  r1  r2  r3  r  rG  rH  rS  rI  rK  rL  rM  rT  rU  r   r   r   test_tt_sls_gradientop2   sF    
z%TestOperators.test_tt_sls_gradientop2simpleZdagZ	async_dag)num_workersnet_typec                    s<  ddl m} ddlm} |dddd dd}tjd	 ttD ]}td| D ]}	d
	|d d|	 }
d
	|d d|	 d }d	|d d|	 }d	|d d|	 d }d
	||	}|j
|
|  dttj dfdttj dfd |j
||  dttj dfdttj dfd j||g| qZqJjddgd jdd dg j j j | _| _jj tt   fddfddt|D }|dd  D ]4}tj|d | jtt |ddd qd S )Nr   )ModelHelper)brewZ
test_modelr[   r3   r2   r     {}_{}r"   z{}_{}_mr  r  )dim_indim_outZweight_initZ	bias_initZ0_0labelZxentr   c                     s   dd l } | jd fddtd D }|D ]T}j|j| j | j	d jdj| j | j	d q2j
j fdd|D }|S )	Nr   rm  c                    s   g | ]}d   |qS )rn  r  )r   rp  )depthr   r   r   _  rr   zCTestOperators.test_dag_net_forking.<locals>.run.<locals>.<listcomp>r2   r   rq  c                    s$   g | ]}j jt |   qS r   )r   r   r   r   )r   
input_blob)input_to_gradrW   r   r   r   h  s   )numpyr   r   r   r   r   r   r   r   rA   r   r  )r   Zinput_blobsrs  Z	gradients)drr  rX   rt  r"  rH   rW   r   r   r   \  s"    z/TestOperators.test_dag_net_forking.<locals>.runc                    s   g | ]
}  qS r   r   r  r   r   r   r   m  rr   z6TestOperators.test_dag_net_forking.<locals>.<listcomp>gg?RV@r   r(  )!Zcaffe2.python.model_helperrj  caffe2.pythonrk  r   r   r   reversedr   r  fcdictr   r  rz   ZSquaredL2Distancer   ZAddGradientOperatorsProtor   ZCopyFromparam_init_nettyperh  r   r   r  r   r   r   r*  r  r/  )rW   ri  rh  rX   rY   rj  rk  itersr   rp  Zbottom_1Zbottom_2Zmid_1Zmid_2topoutputsr  r   )rv  rr  rX   rt  r"  rH   r   rW   r   test_dag_net_forking+  s\    


z"TestOperators.test_dag_net_forking)r/   r0   )r  	slice_dimar   is_emptyc                 C   s   |t |j }|r:tjjdgt|j  tj}|d7 }||j|  }||j|  d }tjt |jtjd}tj	t |jtjdd }	t
||||< t|||	|< tdg ddg}
dd	 }| ||
|||	g| | ||
|||	gddg d S )
Nr   r"   rK   r4  Slice)r  r  endr  c                 S   s0   t |jdkr| S dd t||D }| | fS )Nr   c                 S   s&   g | ]\}}t ||d krdn|qS )r4  N)slice)r   sieir   r   r   r     rr   z?TestOperators.test_slice.<locals>.slice_ref.<locals>.<listcomp>)r+   r}   r  )r    seZslcr   r   r   	slice_ref  s    z+TestOperators.test_slice.<locals>.slice_ref)r+   r}   r   r   r   r   r   r@   r~   r>  minmaxr
   rS   rV   rU   )rW   r  r  r  r   r  rX   rY   Z	start_vecZend_vecrZ   r  r   r   r   
test_slices  s(     zTestOperators.test_slicec                 C   s,   t ddgdg}| |||gdd  d S )NShaper  r}   c                 S   s   | j fS r   r}   r   r   r   r   r=     rr   z*TestOperators.test_shape.<locals>.<lambda>r  )rW   r  rX   rY   rZ   r   r   r   
test_shape  s    zTestOperators.test_shapec                 C   sP   dd }t jjt|jdd }tjddgdg|d}| ||||g| d S )	Nc                    s    fdd|D fS )Nc                    s   g | ]} j | qS r   r  rc  r   r   r   r     rr   zITestOperators.test_shape_with_axes.<locals>.shape_ref.<locals>.<listcomp>r   rh   r   r   r   	shape_ref  s    z5TestOperators.test_shape_with_axes.<locals>.shape_refr3   r   r  r  r}   )axes)	r   r   r  r+   r}   r  r
   rS   rV   )rW   r  rX   rY   r  r  rZ   r   r   r   test_shape_with_axes  s    z"TestOperators.test_shape_with_axesrh   c                 C   sX   t dddgdg}| ||||gdd  t ddgdg}| |||gd	d  d S )
NZHasElementsr    ri   Zhas_elementsc                 S   s   t | dkpt |dkfS rs  r+   rh   r   r   r   r=     rr   z1TestOperators.test_has_elements.<locals>.<lambda>ZIsEmptyr  c                 S   s   t | dkfS rs  r  r   r   r   r   r=     rr   r  )rW   r    ri   rX   rY   rZ   r   r   r   test_has_elements  s    zTestOperators.test_has_elements)initial_iters	max_itersc                 C   s   t d}|dgdg | jdt|gtj	 | jdt|gtj	 t d}|
ddgdg | jdg t d}|t jd||gt dd | j| | jjd  }| |jtj	 | |d	 t|| d S )
Nr  iter	num_iterscriteriastopr  r  r  r   )r
   r  Iterr   r   r   r   r&  r   r:   ru   r{  external_outputr  r  r  r  BlobReferencer   r   r   r   r-   r  )rW   r  r  r  Zcriteria_netr  r~  r   r   r   /test_should_stop_as_criteria_net_execution_step  s(    


z=TestOperators.test_should_stop_as_criteria_net_execution_stepc                 C   s   dd }|dd|dd|ddg}t jd|d	 t d
dt jd|d t ddt d|d g}g d}t d}|t jd|dd | j| t|D ]2\}}| | jj	d
|d   d	 ||  qd S )Nc           	      S   s  d | }d | }td | }|jg |gdgdd |j|g|gdd td	 | }|jg dg|rpd
ndd}|j|g|gdd | j|g td | }|jg |gdgd
d td | }|jg dgd
d}|j|g|gdd | j|g ||||gS )Nzshould_stop_{}	output_{}zinit_{}r"   r  )r}   r  booltozcriterion_{}r   net_{}zender_{})r  r
   r  r  Castr{  r  r  )	r   disabledZshould_stopr  r  	criteriontmpr  enderr   r   r   
createNets  sD    


z>TestOperators.test_disabled_execution_step.<locals>.createNetsr"   Fr2   Trd   Zstep_1r   Zshould_stop_1r  Zstep_2Zshould_stop_2Zstep_3)r   r  r   r  Z	all_stepsr  r  )r
   r  r  r  r  r   r   r   r   r   r  r   )rW   r  netsZstepsexpectedr  r   r   r   r   r   test_disabled_execution_step  s0    -	
z*TestOperators.test_disabled_execution_step)r  r  c                 C   s   t d}|dgdg | jdt|gtj	 t 
d|g}|| t d}|| | j| | jjd  }| |jtj	 | |d ||  d S )Nr  r  r  r  r   )r
   r  r  r   r   r   r   r&  r   r:   ExecutionStepZSetIterr  r  r   r   r   r   r-   )rW   r  r  r  r  r  r~  r   r   r   #test_iter_count_with_execution_step
  s    



z1TestOperators.test_iter_count_with_execution_step)r  r  num_netsc                 C   s  t d}|g dg}| jdt|gtj	 t j
d|d}t|D ]>}t d|}||dgdg t 
d|g}	||	 qP|d t d	}
|
| t d
}|g g d | j| | j|
 | j| | jjd  }| |jtj	 | |d |||   || dkr| jjd  }d}| ||v  | jjd  }| || |t||  d S )Nr  
iter_mutexr  concurrent_stepsr  r  r  Tr  	stats_net)	stats_key	stats_valZstats_tsr   r  s   atomic_iter/stats/iter/num_iterr  )r
   r  ZCreateMutexr   r   r   r   r&  r   r:   r  r   r  Z
AtomicIterZ
AddSubstepZSetConcurrentSubstepsr  r  ZStatRegistryExportr   r   r   r   r-   r  r   r4  )rW   r  r  r  r  r  r  r   r  r  r  r  r~  r  Zatomic_iter_keyZstat_valr   r   r   &test_atomic_iter_with_concurrent_steps  s<    




z4TestOperators.test_atomic_iter_with_concurrent_steps)r  srcdstuse_namec                    s   | |}tjtjg}||v rL |vrL tjurLt }t||j|j} fdd}	t	  }
|rvt
jj|
 }
tjddgdg|
d}| |||gdg | |||g|	\}|  |j d S )Nc                    s   |   gS r   )r   r  r  r   r   r\   V  s    z$TestOperators.test_cast.<locals>.refr  r   rL   r  r   )r   r   rA   r   r  iinfoclipr  r  _NUMPY_TYPE_TO_ENUMr   ZTensorProtoDataTypeNamelowerr
   rS   rT   rV   r   r-   )rW   r  r  r  r  rX   rY   Zftypesinfor\   r  rZ   rP   r   r  r   	test_castF  s    

zTestOperators.test_castg-C6?gGz?)r.   )r  epsa_gradeps_gradc           
   	      s~    fdd}t jddgdg d}| |||gdg | |||g| t jddgdg|d}	| j||	|gddgdd	d
 d S )Nc                    s&   t |  d  } t | d|   fS )Nr   r"   )r   r  r  r  r  r   r   r\   h  s    z%TestOperators.test_logit.<locals>.refZLogitr   rL   r  r   g{Gz?gMb`?)r   r   )r
   rS   rT   rV   rU   )
rW   r  r  r  r  rX   rY   r\   rZ   Zop_gradr   r  r   
test_logita  s    zTestOperators.test_logitT)rj  )r  r  c                    sL    fdd}t jddgdg d}| |||gdg | |||g| d S )Nc                    s   t | } |t | < |fS r   )r   r   isnan)r  rP   r  r   r   r\     s    
z+TestOperators.test_replace_nan.<locals>.refZ
ReplaceNaNr   rL   r  r   rk   )rW   r  r  rX   rY   r\   rZ   r   r  r   test_replace_nan{  s    zTestOperators.test_replace_nanrC   c                 C   s   t jd| t | dS Nr"   )r/   r-   r.   r)   tensorrJ   rK   r   r   r   r=     s   zTestOperators.<lambda>)r  	has_inputhas_extra_shapeextra_shapec                    s   |j j |j t tjkr tj |d|j|g}t  }|r|rntjddgdg||d}	t	|7 qtjddgdg|d}	n tjdg dg|t
d}	g }d
 fdd		}
| ||	|dg | ||	||
\}|  |j  d S )Nr   r  r   rL   )r-   r  r  )r-   r  )r-   r  r}   c                    s   t j d}|gS )Nr}   Z
fill_valuer-   )r   full)rO   r  r-   Zgt_shaper  r   r   r\     s    z-TestOperators.test_constant_fill.<locals>.ref)N)r-   r}  r   r  rq  r}   r  r
   rS   r  r   rT   rV   r   )rW   r  r  r  r  rX   rY   rO   	enum_typerZ   r\   rP   r   r  r   test_constant_fill  s:    
z TestOperators.test_constant_fillc                 C   s   t jd| t | dS r  r  rK   r   r   r   r=     s   c           	         s    j j j t tjkr tjtj dgd g}t }tjdddgdg|d} fdd}| 	|||dg | 
||||\}| |j  d S )	Nr   rK   r  r   VrL   c                    s   t j jd d}|gS )Nr   r  )r   r  r}   )r    r  r  r  r-   r  r   r   r\     s    z9TestOperators.test_constant_fill_from_tensor.<locals>.ref)r-   r}  r   r  rg   rq  r  r
   rS   rT   rV   r   )	rW   r  rX   rY   rO   r  rZ   r\   rP   r   r  r   test_constant_fill_from_tensor  s     z,TestOperators.test_constant_fill_from_tensor)r  rH   rv  c                    s  ddl m}m} tjd |jdd}| j	g d d| _
| j	dd	g |j|d
d	||dd |jd	dgd	g |jd	gdg |j jD ]}t| qtj| jddi\}}	dd t|	D }	td}
|
 jd d = |
 j	| |	d dksJ g d}t| \}}}g d}t| \}}}|
 j	dg |
 j	| j |
 j	| j g d dg}tjd g dddgddgddgdg fdd|D |||||| fd d|jD | |
 dgd!}td"tj|||tj tdtjd||tj td#tjj d||fd$tj! d%d& }| j"t#j$|d'd dD |g d(d) d*D ]&}| %t#j$|d+d dD |dg qd S ),Nr   )model_helperrk  rm  ZElmanrl  )input_tseq_lengthsZtimestephidden_t_prev	gates_t_w	gates_t_brg  hidden_tZgates_tr  r2   )ro  rp  r   r  hidden_t_gradc                 S   s   i | ]\}}t |t |qS r   )r   )r   r  r  r   r   r   
<dictcomp>  s   z>TestOperators.test_elman_recurrent_network.<locals>.<dictcomp>ZElmanBackwardgates_t_grad))r  hiddenr   )r  r  r"   )r  r  r   ))Zhidden_t_prev_gradhidden_gradr   )r  r  r"   )r  Z
input_gradr   )r  r  r  r  hidden_inputr  ZRecurrentNetwork)r  r  hidden_outputZstep_workspacesr  r  r  r"   r4  c                    s   g | ]}  |qS r   r4  rc  rO   r   r   r   	  s   z>TestOperators.test_elman_recurrent_network.<locals>.<listcomp>c                    s   g | ]}  |qS r   r  )r   r   r  r   r   r   	  rr   )Z	alias_srcZ	alias_dstZalias_offsetZrecurrent_statesZinitial_recurrent_state_idslink_internallink_externallink_offsetbackward_link_internalbackward_link_externalbackward_link_offsetr   step_netbackward_step_netZoutputs_with_gradsr  r  r  c                 S   s  | j d }| j d }| j d }tj|d ||fd}|j d |d ksJJ |j d |ks\J |j d |ksnJ ||dd d d d f< t|D ]b}	| |	 d||}
||	 d||}t||j}|d|||
d|| }t|||	d < q|dd  ||d d||fS )Nr   r"   r2   r  r4  )r}   r   r~   r   rM  dotr   r!   )r  r  Zgates_wZgates_br  r   r   r   r  r  r  r  Zgatesr   r   r   ry  &	  s    


z=TestOperators.test_elman_recurrent_network.<locals>.referencec                 S   s   g | ]}t |qS r   r   r  r   r[   r   r   r   r   ;	  s   )r   r"   r2   )Zoutputs_to_check)r   r2   rd   c                 S   s   g | ]}t |qS r   r  r  r   r   r   r   E	  s   )&rw  r  rk  r   r   r   rj  r{  Zexternal_inputr  r}  r  ry  r  rz   ZSigmoidr|  rZ   r   ZRunOperatorOncer
   ZGradientRegistryZGetBackwardPassr   r  r  rS   paramsr  r   r   rA   r  r@   rV   r)   rz  rU   )rW   r  rH   rv  r  rk  r  rZ   Zbackward_opsZbackward_mappingr  linksr  r  r  Zbackward_linksr  r  r  Zrecurrent_inputsry  r   r   r  r   test_elman_recurrent_network  s    






	z*TestOperators.test_elman_recurrent_network)Zsuppress_health_checkr   )rH   chr>  pad
block_sizec	                 C   s   t |d|  | dk t |d|  | dk tj||||tj}	tjddgdg||d}
| ||
|	gdg | 	||
|	gddg d S )Nr2   r   ZSpaceToBatchr   rL   r  r  
r   r   r   r   r   rA   r
   rS   rT   rU   rW   rH   r  r  r>  r  r  rX   rY   r   rZ   r   r   r   test_space_to_batchK	  s    	z!TestOperators.test_space_to_batchc	                 C   s   t |d|  | dk t |d|  | dk tj|| | ||d|  | |d|  | tj}	tjddgdg||d}
| ||
|	gdg | 	||
|	gddg d S )Nr2   r   ZBatchToSpacer   rL   r  r  r   r   r   r   test_batch_to_space\	  s    	
z!TestOperators.test_batch_to_spaceg       r  )r   rc   scalec                 C   sL   t jddg|sdndg|d}| |||gdg | |||gddg d S )NZScaler   rL   )r  r   r{   )rW   r   rc   r  rX   rY   rZ   r   r   r   
test_scaleq	  s    zTestOperators.test_scale)r  c                 C   sv   | dd}| jd| | jjd d}| jd| | || jjd   | || jjd   d S )Nasciiignorer  r   )	encoder   r   r   r   	serializeZdeserializer   r   )rW   r  Z
serializedr   r   r   test_string_serde}	  s    zTestOperators.test_string_serder  r  )ZconstantZreflectedge)r  r   input_channels
batch_sizer  modec	                    s   t |k tjddgdg |d}	|dkr\tj||||tjd }
 fdd}n,tj||||tjd }
 fd	d}| ||	|
g| | 	||	|
gd
g | 
||	|
gd
d
g d S )NPadImager   rL   )r  r  r  r  r  c                    s    t | dffdf fS N)r   r   r   r  r   r  r  r   r   numpy_pad_ref	  s    z8TestOperators.test_same_pad_image.<locals>.numpy_pad_refc                    s    t | ddfff fS r  r  r   r  r   r   r  	  s    r   )r   r
   rS   r   r   r   r   rA   rV   rT   rU   )rW   r  r   r  r  r  r  rX   rY   rZ   r   r  r   r  r   test_same_pad_image	  s:    
z!TestOperators.test_same_pad_image)	pad_tpad_lpad_bpad_rr   r  r  r  r  c                    s   t |tk tjddgdg |d	}|dkrrtj||||tjd } fdd}n2tj||||tjd } fd	d}| 	|
||g| | 
|||gd
g | |
||gd
d
g d S )Nr  r   rL   )r  r  r  r  r  r  r  r  c                    s    t | dffdf fS r  r  r   r  r  r  r  r  r   r   r  	  s    z3TestOperators.test_pad_image.<locals>.numpy_pad_refc                    s    t | ddfff fS r  r  r   r  r   r   r  	  s    r   )r   r  r
   rS   r   r   r   r   rA   rV   rT   rU   )rW   r  r  r  r  r   r  r  r  r  rX   rY   rZ   r   r  r   r  r   test_pad_image	  s@    zTestOperators.test_pad_imager   )r   r  r  r  r*  c                    sT  t jdg ddg|d}tjd tjtjd }	tjtjd }
tj tjd }|dkr|dd	d	d
} fddfdd}| 	||||	|
g|dkrʈn| t
j }d|fd|	fd|
fg}|D ]\}}||| qtdD ]}|| q|D ]"\}}tj|j|  | q,d S )NZInstanceNorm)r   r  biasrL   )r  r*  rm  r  r  r"   r2   rd   c                    s   |     } | | dd d tjf  }|t| d d d tjf  }|  }|| ddd }|| ddd }|fS r   )rM  meanr   Znewaxisr0  rG  r    r  r  ri   )r  r*  r  r   r   r   ref_nchw	  s    &z2TestOperators.test_instance_norm.<locals>.ref_nchwc                    s:   |  dd dd}  | ||d }| dd ddfS )Nr2   rd   r"   r   )swapaxesr  )r  r   r   ref_nhwc	  s    z2TestOperators.test_instance_norm.<locals>.ref_nhwcr  r   r  r  re   )r
   rS   r   r   r   r   r   rA   r  rV   r   r   r   r   r   r   r   r   r   r   r   )rW   r   r  r  r  r*  rX   rY   rZ   r  r  r   r  r   ZfeedsZblobr  r   r   )r  r*  r  r  r   r   test_instance_norm	  sB    		
z TestOperators.test_instance_norminpc                 C   s   t t| | dS )N)r.   r-   )r?   r)   rJ   )dtr   r   r   r=   
  s   
c                 C   sx  |\}}}|j t dkr@|jtjtjhv r@td|j  d S |jtjtjhv r^|tj	}|j t dt dfv rt
t|  tdjk   t
t| tj tdjk  t|f|jdd   |j }tdg ddg}tdd	d
gdg}	dd }
dd }| |||||g|
 | ||	||g| |j tjkrt| |||||gddg d S )NrA   z
Bailout {}r:   r@   r"   ZSparseToDense)Ir   r   rL   r#  r   c                 S   s>   t j|j|jd}t| D ]\}}||  || 7  < q|gS r  )r   r~   r}   r-   r   )r#  r   r   Or   r   r   r   r   sparse_to_dense-
  s    z;TestOperators.test_sparse_to_dense.<locals>.sparse_to_densec                 S   sV   t jt | d f|jdd   |jd}t| D ]\}}||  || 7  < q2|gS r  )r   r~   r  r}   r-   r   )r#  r   r$  r   r   r   r   r   sparse_to_dense_noshapeinfer3
  s    *zHTestOperators.test_sparse_to_dense.<locals>.sparse_to_dense_noshapeinferr   )r-   r   r   r   CUDAZHIPr  r  r   r@   r   r   Zravelr  r  allr:   r  r~   r}   r
   rS   rV   rA   rU   )rW   r!  rX   rY   Z	first_dimr   r#  r   rZ   Zop_noshapeinferr%  r&  r   r   r   test_sparse_to_dense
  s$    
"&*"z"TestOperators.test_sparse_to_densec                 C   s   |\}}t dddgd}dd }| ||||g| | ||||gdg | ||||gddg | ||||gddg d S )	NZ
DotProductr   rL   rP   c                 S   s   t j| | ddfS Nr"   r   )r   r  r   r   r   r   
dotproductD
  s    z2TestOperators.test_dot_product.<locals>.dotproductr   r"   )r
   rS   rV   rT   rU   )rW   rO   rX   rY   r   rL   rZ   r+  r   r   r   test_dot_product>
  s    zTestOperators.test_dot_product)r   Mr   	pad_valuec                    s   t jt jd }t j t jd }tjdddgdd}	 fdd}
| ||	||g|
 | ||	||gd	g | 	||	||gd	d	g | 	||	||gd
d	g d S )Nr  DotProductWithPaddingr   rL   rP   )r.  c                    s|   t t ft j } k rP| |d d d f< t j|| ddfS ||d d d  f< t j||  ddfS d S r*  )r   r>  r  r   rA   r  )r   rL   Zr   r-  r   r.  r   r   r+  X
  s     z?TestOperators.test_dot_product_with_padding.<locals>.dotproductr   r"   
r   r   r   r   rA   r
   rS   rV   rT   rU   )rW   r   r-  r   r.  rX   rY   r   rL   rZ   r+  r   r1  r   test_dot_product_with_paddingL
  s    	z+TestOperators.test_dot_product_with_padding)r   r-  r.  c           
         s   d  t j|t jd }t j| t jd }tjdddgdd|d} fd	d
}	| ||||g|	 | ||||gdg | 	||||gddg | 	||||gddg d S )Nr2   r  r/  r   rL   rP   T)Z	replicater.  c                    sd   dd l m}  k r:|| d  }tj|| ddfS ||d  }tj||  ddfS d S )Nr   r"   r   )Znumpy.matlibZmatlibZrepmatr   r  )r   rL   Znpmr0  r   r-  r   r   r+  s
  s    zCTestOperators.test_dot_product_with_rep_padding.<locals>.dotproductr   r"   r2  )
rW   r   r-  r.  rX   rY   r   rL   rZ   r+  r   r4  r   !test_dot_product_with_rep_paddingf
  s    	z/TestOperators.test_dot_product_with_rep_padding)r   r-  c                 C   s   t j||t jd }tddgd}| |||gdd  | |||gdg t j||t jd }tddgd}| |||gdd  | |||gdg d S )	Nr  ZEnsureDenser   c                 S   s   | gS r   r   r   r   r   r   r=   
  rr   z1TestOperators.test_ensure_dense.<locals>.<lambda>r   rP   c                 S   s   | gS r   r   r   r   r   r   r=   
  rr   )	r   r   r   r   rA   r
   rS   rV   rT   )rW   r   r-  rX   rY   r   rZ   r   r   r   test_ensure_dense
  s    zTestOperators.test_ensure_dense)r   r-  num_bucketsc           	         sz   t j t j}d\tjddgddgd} fdd}| |||gd	d
g | |||g| d S )N)r  r  ZAccumulateHistogramr   cur_histacc_hist)lower_boundupper_boundr7  c                    s   t jd ft jd}  }t j ft jd}d|| k < d || k< | | k| k @   | d t j|| k| k @ < t|jd D ]0}t|jd D ]}||| |   d7  < qq|| }}||gS )Nr2   rK   r   r"   )r   r~   r@   r   r   r}   )r   histsegmentrL   r   rp  r8  r9  r-  r   r:  r7  r;  r   r   	histogram
  s$    
z=TestOperators.test_accumulate_histogram_op.<locals>.histogramr   r"   )	r   r   r   r   rA   r
   rS   rT   rV   )	rW   r   r-  r7  rX   rY   r   rZ   r?  r   r>  r   test_accumulate_histogram_op
  s    
z*TestOperators.test_accumulate_histogram_oprR   )queue_capacity
time_sleepnum_blobs_to_equeuenum_blobs_to_dequeuec                    s    fdd}t d}d| _|jg d||d}t }||  t d}	d|	 _|		|g|}
||	}t
j||gd}|  | t  |  |  W d	   n1 s0    Y  d	S )
a8  
        Tests SafeDequeueBlobsOp being cancellable.

        Create a queue with the number of BlobsQueue less than the number
        SafeDequeueBlobs to cause the hanging behavior when running the Net.

        Then call cancel from the previous sleeping thread to ensure exception
        is raised.
        c                    s   t   |   d S r   )r  sleepcancel)net_instancerB  r   r   _net_instance_cancel
  s    
z^TestOperators.test_safe_dequeue_blob__raises_exception_when_hang.<locals>._net_instance_cancelr  Zasync_schedulingZ
queue_namer  r  r  N)r
   r  r{  r}  r  r   r   Z
create_netr   r  r  r  r  r   r   r  )rW   rA  rB  rC  rD  rI  r  r  r   r  r   rG  r  r   rH  r   2test_safe_dequeue_blob__raises_exception_when_hang
  s(    




z@TestOperators.test_safe_dequeue_blob__raises_exception_when_hang)}__name__
__module____qualname__rx   r   r)   rI   r'   rm   r_   r   r|   rv   r   r   r   r   r   r   r  r   r   r   r   r(   rB   Zexpanded_device_optionsr   r;   r   unittestZskipIfr   has_gpu_supportZfloatsr   r  r
  r,   r  r   rA   r   r@   r  r  r(  r3  staticmethodrD  rL  rP  rQ  rZ  r[  r]  r_  Z
gcs_no_hipri  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r"  r,  rD  rV  r^  r`  re  rf  r  r  r  r  r  r  r  r  r  r   r   r  r  r  r  rE   r:   r>   r  r  r  r	   Zfilter_too_muchr  r  r  textr	  r  r  r   r)  r,  r3  r5  r6  r@  rJ  r   r   r   r   ro   ~   s  	 
 


	A





&
!

"

	
!+













P




:



V"+
'
+
*
"
*B

E



$


& 

r




	










 





	

%

0%



ro   __main__)r"   r#   )N)Bru  r   r   r  	functoolsr   r   Zfuture.utilsr   r   Z
hypothesisr   r   r   r	   Zhypothesis.strategies
strategiesr'   rN  r  rw  r
   r   r   r   Z"caffe2.python.hypothesis_test_utilpythonZhypothesis_test_utilr)   Zcaffe2.protor   ZInitOpsLibraryrO  r   r!   Z	compositer1   rA   r?   r  FLOATr@   ZINT32r  BOOLZuint8ZUINT8Zint8ZINT8Zuint16ZUINT16int16ZINT16r:   ZINT64r   DOUBLEr  rE   r_   ra   rn   ZHypothesisTestCasero   rK  mainr   r   r   r   <module>   s|   















                     
