a
    ==icb                     @   s   d dl mZmZ d dlmZ d dlmZmZ d dl	Z
d dlZd dlZd dlZd dlZd dlmZmZmZ d dlm  mZ d dlmZ ed ed ed ed	 ed
 dZG dd dZG dd dejZedkrd dlZe   dS )    )givensettingsN)ProcessQueue)core	workspacedyndep)IoErrorz2@/caffe2/caffe2/distributed:file_store_handler_opsz3@/caffe2/caffe2/distributed:redis_store_handler_opsz%@/caffe2/caffe2/distributed:store_opsz%@/caffe2/caffe2/contrib/gloo:gloo_opsz)@/caffe2/caffe2/contrib/gloo:gloo_ops_gpuZGLOOc                   @   s   e Zd Zdd Zdd ZdS )TemporaryDirectoryc                 C   s   t  | _| jS N)tempfilemkdtemptmpdir)self r   n/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/caffe2/contrib/gloo/gloo_test.py	__enter__   s    
zTemporaryDirectory.__enter__c                 C   s   t | j d S r   )shutilrmtreer   )r   typevalue	tracebackr   r   r   __exit__#   s    zTemporaryDirectory.__exit__N)__name__
__module____qualname__r   r   r   r   r   r   r
      s   r
   c                   @   s  e Zd ZdZdZd:ddZd;ddZd<ddZd=d	d
Zd>ddZ	e
ejdddejededdejdddeejge dedddd Zd?ddZd@ddZe
ejdddejededdejdddeejge dedddd ZdAd d!Ze
ejdddejededdejdddeejge deddd"d# ZdBd$d%Ze
ejdddejededdejdddeejge ded&dd'd(d) Ze
eejgd*eddd+d, ZdCd-d.Ze
ejdddeejgd/eddd0d1 ZdDd2d3Ze
ejdddeejgd/eddd4d5 ZdEd6d7Z e
ejdddeejgd/eddd8d9 Z!dS )FTestCaser   Nc           	         s   t   fdd}g }t|d D ]*}||d< t||d}|  || q&t|dkr|d}| r||d qh| 	
 d  }t|tr|qR| | qRd S )	Nc               
      s   zNt  0 | i | t  d W d    n1 sB0    Y  W n. ty| } z| W Y d }~n
d }~0 0 d S )NT)r   DeviceScoper   ResetWorkspaceput	Exception)argskwargsexdevice_optionfnqueuer   r   run_fn0   s    ,z)TestCase.run_test_locally.<locals>.run_fn	comm_size	comm_rank)targetr"   r   
   zJob failed without a result)r   ranger   startappendlenpopis_alivejoinZassertFalseemptyget
isinstancer    
assertTrue)	r   r&   r%   r"   r(   Zprocsiprocor   r$   r   run_test_locally+   s(    


zTestCase.run_test_locallyc                 K   s   t d}| | t d}| | t||d< t||d< t|& |f i | t  W d    n1 sv0    Y  d S )N	COMM_RANKZ	COMM_SIZEr*   r)   )osgetenvZassertIsNotNoneintr   r   r   r   )r   r&   r%   r"   r*   r)   r   r   r   run_test_distributedU   s    



zTestCase.run_test_distributedc           	      C   s   d}|d u rxt d}tt dd}|d urXttjdg |gttj	d ||d nttjdg |g|d	 d
}nt|d }|d urttjd|g|gdt
d n"ttjd|g|g||dt
d ||fS )Nstore_handlerZ
REDIS_HOSTZ
REDIS_PORTi  ZRedisStoreHandlerCreate/)prefixhostportZFileStoreHandlerCreate)pathcommon_worldz.forkedZCloneCommonWorldT)syncengineZCreateCommonWorld)sizeZrankrH   rI   )r=   r>   r?   r   RunOperatorOncer   CreateOperatorstrr   test_counter	op_engine)	r   r*   r)   r   existing_cwrA   Z
redis_hostZ
redis_portrG   r   r   r   create_common_world`   s^    
		zTestCase.create_common_worldc                 C   sv   t  jd7  _dt j}|dkrNt|t| tt	d||gg  ntt	d|g|g t
t|S )N   zsync_{}r   ZStoreSetZStoreGet)r   sync_counterformatr   FeedBlobpickledumpsrK   r   rL   loads	FetchBlob)r   rA   r   r*   blobr   r   r   synchronize   s&    zTestCase.synchronizeFc                 C   s   | j |||d\}}| j|||d}| j|||d}t|D ]}	g }
t|D ]J}d|}|| | }t|||rxtjntj}t	|| |

| qLtd}|j|g|
 |
|	td t| t|  t|D ] }tjt|
| |	|  qtdD ]}t|  qq<d S )Nr*   r)   r   r*   blob_{}	broadcast)rootrI      )rQ   r[   r-   rT   npfullfloat16float32r   rU   r/   r   NetZ	BroadcastrO   	CreateNetRunNetNametestingassert_array_equalrY   )r   r*   r)   	blob_size	num_blobsr   use_float16rA   rG   r8   blobsjrZ   offsetr   net_tmpr   r   r   _test_broadcast   sR    



zTestCase._test_broadcast      )	min_value	max_valueg     @@g    .ArR   ra   )r)   rl   rm   r%   rn   i'  )deadlinec              
   C   sx   t  jd7  _tdd ur4| j| j||||d n@t *}| j| j||||||d W d    n1 sj0    Y  d S NrR   r<   )rl   rm   rn   r%   )r)   rl   rm   r%   r   rn   )r   rN   r=   r>   r@   rt   r
   r;   r   r)   rl   rm   r%   rn   r   r   r   r   test_broadcast   s&    zTestCase.test_broadcastc                 C   s  | j |||d\}}| j|||d}| j|||d}g }	t|D ]F}
d|
}t||| |
 |rhtjntj}t	|| |	
| q@td}|j|g|	 |	td t| t|  t|D ]0}
tjt|	|
 || || d  d  qtdD ]}t|  qd S )	Nr\   r]   r^   	allreducerI   rR   ru   ra   )rQ   r[   r-   rT   rb   rc   rd   re   r   rU   r/   r   rf   	AllreducerO   rg   rh   ri   rj   rk   rY   )r   r*   r)   rl   rm   r   rn   rA   rG   ro   r8   rZ   r   rr   rs   r   r   r   _test_allreduce   sL    



zTestCase._test_allreducec                 C   s   | j |||d\}}| j ||||d\}}td}d}	||fD ]}
g }t|	D ]<}d|}t|||	 | tj}t|| |	| qNt
d}|j|
g| |td t| t|	D ]0}tjt|| |	| |	| d  d	  qq>d S )
Nr\   )r*   r)   r   rP   g     @ra   r^   Zallreduce_multicwr~   rR   ru   )rQ   r?   r-   rT   rb   rc   re   r   rU   r/   r   rf   r   rO   Z
RunNetOncerj   rk   rY   )r   r*   r)   r   Z_store_handlerrG   _Zcommon_world2rl   rm   Zcwro   r8   rZ   r   rr   r   r   r   _test_allreduce_multicw#  s@    




z TestCase._test_allreduce_multicwc              
   C   sx   t  jd7  _tdd ur4| j| j||||d n@t *}| j| j||||||d W d    n1 sj0    Y  d S rz   )r   rN   r=   r>   r@   r   r
   r;   r{   r   r   r   test_allreduceJ  s&    zTestCase.test_allreducec              	   C   s  | j |||d\}}| j|||d}| j|||d}g }	t|D ]F}
d|
}t||| |
 |rhtjntj}t	|| |	
| q@tj|tjd}|}|| d | }t|D ]&}
t||||
< ||kr|| nd}qd}t	|| |	
| td}|j|g|	 |	td	 t| t|  t|D ]>}
tjtt|	|
 || || || d  d
  q6tdD ]}t|  q~d S )Nr\   r]   r^   )ZdtyperR   r   Z
recvCountsZreduce_scatterr~   ru   ra   )rQ   r[   r-   rT   rb   rc   rd   re   r   rU   r/   Zzerosint32minr   rf   ZReduceScatterrO   rg   rh   ri   rj   rk   resizerY   )r   r*   r)   rl   rm   r   rn   rA   rG   ro   r8   rZ   r   Zrecv_counts	remaining
chunk_sizeZrecv_counts_blobrr   rs   r   r   r   _test_reduce_scattere  s^    




zTestCase._test_reduce_scatterc              
   C   sx   t  jd7  _tdd ur4| j| j||||d n@t *}| j| j||||||d W d    n1 sj0    Y  d S rz   )r   rN   r=   r>   r@   r   r
   r;   r{   r   r   r   test_reduce_scatter  s&    zTestCase.test_reduce_scatterc                 C   sN  | j |||d\}}| j|||d}| j|||d}g }	t|D ]F}
d|
}t||| |
 |rhtjntj}t	|| |	
| q@td}|j|g|	 dgtd t| t|  tg }t|D ]B}
t|D ]4}t||
| | |rtjntj}t||f}qqtjtd| tdD ]}t|  q4d S )Nr\   r]   r^   Z	allgatherZGatheredr~   ra   )rQ   r[   r-   rT   rb   rc   rd   re   r   rU   r/   r   rf   Z	AllgatherrO   rg   rh   ri   arrayZconcatenaterj   rk   rY   )r   r*   r)   rl   rm   r   rn   rA   rG   ro   r8   rZ   r   rr   Zexpected_outputrp   rs   r   r   r   _test_allgather  sV    





zTestCase._test_allgatherr,   )Zmax_examplesry   c              
   C   sx   t  jd7  _tdd ur4| j| j||||d n@t *}| j| j||||||d W d    n1 sj0    Y  d S rz   )r   rN   r=   r>   r@   r   r
   r;   r{   r   r   r   test_allgather  s&    zTestCase.test_allgatherr%   c                 C   sl   t  jd7  _tdd ur.| j| j|d n:t $}| j| jd||d W d    n1 s^0    Y  d S )NrR   r<   r   ru   r)   r%   r   )r   rN   r=   r>   r@   r   r
   r;   )r   r%   r   r   r   r   test_forked_cw  s    
zTestCase.test_forked_cwc                 C   sh   | j |||d\}}td}|j|gg td t| t|  t	dD ]}t|  qPd S )Nr\   Zbarrierr~   ra   )
rQ   r   rf   BarrierrO   r   rg   rh   ri   r-   )r   r*   r)   r   rA   rG   rr   rs   r   r   r   _test_barrier%  s    


zTestCase._test_barrier)r)   r%   c                 C   sl   t  jd7  _tdd ur.| j| j|d n:t $}| j| j|||d W d    n1 s^0    Y  d S NrR   r<   r   r   )r   rN   r=   r>   r@   r   r
   r;   r   r)   r%   r   r   r   r   test_barrier=  s    zTestCase.test_barrierc           	      C   s   ||d kf}| j |||d\}}td}|sD|j|gg td n&|j|g|gtd ddl}|d t	| t
|  dS )z
        One node calls close connection, others wait it on barrier.
        Test will check that all will exit eventually.
        ru   r\   Zbarrier_or_closer~   r   Ng?)rQ   r   rf   r   rO   ZDestroyCommonWorldtimesleepr   rg   rh   ri   )	r   r*   r)   r   closerrA   rG   rr   r   r   r   r   _test_close_connectionN  s$    




zTestCase._test_close_connectionc                 C   s   dd l }|  }t jd7  _tdd ur>| j| j|d n:t $}| j| j|||d W d    n1 sn0    Y  | 	|  | d d S )Nr   rR   r<   r   r   g      4@)
r   r   rN   r=   r>   r@   r   r
   r;   Z
assertLess)r   r)   r%   r   
start_timer   r   r   r   test_close_connectionp  s     $zTestCase.test_close_connectionc                 C   s   | j |||d\}}|dkrd}d}g }t|D ]<}	d|	}
t||| |	 tj}t|
| ||
 q0t	
d}|j|g| |td t| t|  dS )	zV
        Only one node will participate in allreduce, resulting in an IoError
        r\   r   i  rR   r^   r}   r~   N)rQ   r-   rT   rb   rc   re   r   rU   r/   r   rf   r   rO   rg   rh   ri   )r   r*   r)   r   rA   rG   rl   rm   ro   r8   rZ   r   rr   r   r   r   _test_io_error  s0    	



zTestCase._test_io_errorc              	   C   s   t  jd7  _| tj tdd ur:| j| j|d n:t $}| j	| j|||d W d    n1 sj0    Y  W d    n1 s0    Y  d S r   )
r   rN   assertRaisesr	   r=   r>   r@   r   r
   r;   r   r   r   r   test_io_error  s    zTestCase.test_io_error)N)N)NN)N)NNNNNF)NNNNNF)NNN)NNNNNF)NNNNNF)NNN)NNN)NNN)"r   r   r   rN   rS   r;   r@   rQ   r[   rt   r   stZintegersr?   Zsampled_fromhuZcpu_doZbooleansr   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   '   s   
*

/
      
5      
3   
'      
>      
6
   
   
"   
$r   __main__)!Z
hypothesisr   r   Zhypothesis.strategies
strategiesr   multiprocessingr   r   numpyrb   r=   rV   r   r   Zcaffe2.pythonr   r   r   Z"caffe2.python.hypothesis_test_utilpythonZhypothesis_test_utilr   Zgloo.pythonr	   ZInitOpsLibraryrO   r
   ZHypothesisTestCaser   r   Zunittestmainr   r   r   r   <module>   s4   




	     