a
    +=ich                     @   sJ  d Z ddlZddlZddlZddlm  mZ ddl	m
Z
 ddlZddlmZ ddlmZ ddlmZ ddlmZ G dd	 d	ejjZG d
d dejje
jZG dd dejje
jZG dd dejje
jZG dd dejje
jZG dd dejje
jZG dd dejje
jZG dd dejje
jZedkrFejj j!"  dS )zKTests for tf.keras models with callbacks, checkpointing with dist
strategy.    N)parameterized)losses)distribute_strategy_test)distributed_training_utils_v1)optimizer_combinationsc                   @   s    e Zd ZdZdd Zdd ZdS )CounterzCounts the number of times each callback method was run.

    Attributes:
      method_counts: dict. Contains the counts of time  each callback method was
        run.
    c              
   C   s<   t t| _g d}|D ]}t| || |t| | qd S )N)on_batch_beginon_batch_endon_epoch_beginon_epoch_endon_predict_batch_beginon_predict_batch_endon_predict_beginon_predict_endon_test_batch_beginon_test_batch_endon_test_beginon_test_endon_train_batch_beginon_train_batch_endon_train_beginon_train_end)collectionsdefaultdictintmethod_countssetattrwrap_with_countsgetattr)selfZmethods_to_countmethod_name r!   r/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/keras/distribute/keras_utils_test.py__init__(   s    zCounter.__init__c                    s    fdd}|S )Nc                     s    j   d7  <  | i |S )N   )r   )argskwargsmethodr    r   r!   r"   _call_and_countD   s    z1Counter.wrap_with_counts.<locals>._call_and_countr!   )r   r    r(   r)   r!   r'   r"   r   C   s    zCounter.wrap_with_countsN)__name__
__module____qualname____doc__r#   r   r!   r!   r!   r"   r       s   r   c                   @   s   e Zd Zejjjejjj	e
 dd Zejjjejjj	e
 dd Zejjjejjj	e
 dd ZdS )%TestDistributionStrategyWithCallbacksc           
      C   s   |  * t }|jdddgd W d    n1 s80    Y  t|}t }d}d}d}|j|||d|||gd	 t|tj	j
jjjrt s|jj}|| }	|| r|	d
7 }	n|}	| |j||	 ||	 |||| || ||||	 ||	 d
d
d d S )Nsgdmsemae	optimizerlossmetrics         r   )epochssteps_per_epochverboseZvalidation_datavalidation_steps	callbacksr$   )r   r	   r
   r   r   r   r   r   r   r   r   r   )scopekeras_test_lib	get_modelcompileget_datasetr   fit
isinstancetfcompatv1
distributeexperimentalTPUStrategyexecuting_eagerlyextendedsteps_per_runassertDictEqualr   )
r   distributionmodeldatasetcounterr9   r:   r<   rM   Znum_batch_call_per_epochr!   r!   r"   test_callbacks_in_fitN   sX    
0
	
z;TestDistributionStrategyWithCallbacks.test_callbacks_in_fitc                 C   s   |  * t }|jdddgd W d    n1 s80    Y  t|}t }|j|d|gd | |jddddd d S )	Nr/   r0   r1   r2   r7   stepsr=   r$   )r   r   r   r   )	r>   r?   r@   rA   rB   r   evaluaterN   r   r   rO   rP   rQ   rR   r!   r!   r"   test_callbacks_in_eval   s    
0
z<TestDistributionStrategyWithCallbacks.test_callbacks_in_evalc                 C   s   |  * t }|jdddgd W d    n1 s80    Y  t|}t }|jt|d|gd | |j	ddddd d S )	Nr/   r0   r1   r2   r7   rT   r$   )r   r   r   r   )
r>   r?   r@   rA   rB   r   predictget_predict_datasetrN   r   rW   r!   r!   r"   test_callbacks_in_predict   s$    
0
z?TestDistributionStrategyWithCallbacks.test_callbacks_in_predictN)r*   r+   r,   rE   __internal__rH   combinationsgeneratetesttimesr?   all_strategy_combinationsrS   rX   r[   r!   r!   r!   r"   r.   K   s$   


8




r.   c                   @   sH  e Zd Zejjjejjjj	ejjjj
gdgddd Zejjjejjjj	ejjjj
gddgddd Zejjjejjjj	ejjjj
gddgddd	 Zejjjejjjj	ejjjj
ejjjjgddgdd
d Zejjjejjjj	ejjjj
ejjjjgddgddd Zejjje dd ZdS )"TestDistributionStrategyErrorCasesgraphrO   modec              
   C   s   |    tjdd }||}| tdF |  t||d  W d    n1 s\0    Y  W d    n1 sz0    Y  W d    n1 s0    Y  d S )Nc                  S   sB   t j } | jjdr(t ddggS t ddgddggS d S )NGPU:0r$   r6   )rE   rH   get_replica_contextreplica_id_in_sync_groupdeviceendswithconstantctxr!   r!   r"   run   s    
ziTestDistributionStrategyErrorCases.test_validating_dataset_input_tensors_with_shape_mismatch.<locals>.runzLInput tensor shapes do not match for distributed tensor inputs PerReplica:.+	cached_sessionrE   functionrn   assertRaisesRegex
ValueErrorr>   r   Z#validate_distributed_dataset_inputsr   rO   rn   xr!   r!   r"   9test_validating_dataset_input_tensors_with_shape_mismatch   s    



z\TestDistributionStrategyErrorCases.test_validating_dataset_input_tensors_with_shape_mismatcheagerc              
   C   s   |    tjdd }||}| tdF |  t||d  W d    n1 s\0    Y  W d    n1 sz0    Y  W d    n1 s0    Y  d S )Nc                  S   sH   t j } | jjdr.t jddggt jdS t jddggt jdS d S )Nrf   r$   r6   Zdtype)	rE   rH   rg   rh   ri   rj   rk   int32float64rl   r!   r!   r"   rn      s    
ziTestDistributionStrategyErrorCases.test_validating_dataset_input_tensors_with_dtype_mismatch.<locals>.runzLInput tensor dtypes do not match for distributed tensor inputs PerReplica:.+ro   rt   r!   r!   r"   9test_validating_dataset_input_tensors_with_dtype_mismatch   s    



z\TestDistributionStrategyErrorCases.test_validating_dataset_input_tensors_with_dtype_mismatchc           	   
   C   s  |    | B t }tjjjd}d}dg}|j	|||d W d    n1 s\0    Y  t
|}| t& |j|dddddd	 W d    n1 s0    Y  tjd
}| t$ |j|ddd|d W d    n1 s0    Y  | }| t  |j|ddd W d    n1 s40    Y  | t |j|dd W d    n1 sn0    Y  | t |j|dd W d    n1 s0    Y  W d    n1 s0    Y  d S )NMbP?r0   r1   r5   r$   r6   r   g      ?)r9   r:   r;   Zvalidation_splitr<   )
   )r9   r:   r;   sample_weight)r9   r;   )r;   )rp   r>   r?   r@   rE   rF   rG   trainGradientDescentOptimizerrA   rB   assertRaisesrs   rC   nprandomrepeatrV   rY   )	r   rO   re   rP   r3   r4   r5   rQ   r   r!   r!   r"   test_unsupported_features  sB    	
.
$
$
0.z<TestDistributionStrategyErrorCases.test_unsupported_featuresc              	   C   s   |  t G dd dtj}|d}t sd| td |d W d    qn1 sX0    Y  n
|d W d    n1 s0    Y  d S )Nc                       s$   e Zd Z fddZdd Z  ZS )zeTestDistributionStrategyErrorCases.test_distribution_strategy_on_subclassed_model.<locals>._SimpleMLPc                    s   t    tj|| _d S N)superr#   keraslayersDensedense)r   Z
num_labels	__class__r!   r"   r#   O  s    
znTestDistributionStrategyErrorCases.test_distribution_strategy_on_subclassed_model.<locals>._SimpleMLP.__init__c                 S   s
   |  |S r   )r   )r   inputsr!   r!   r"   callS  s    zjTestDistributionStrategyErrorCases.test_distribution_strategy_on_subclassed_model.<locals>._SimpleMLP.call)r*   r+   r,   r#   r   __classcell__r!   r!   r   r"   
_SimpleMLPN  s   r   r8   We currently do not support distribution strategy with a `Sequential` model that is created without `input_shape`/`input_dim` set in its first layer or a subclassed model.r/   )r>   r   ModelrE   rK   rr   rs   rA   )r   rO   r   rP   r!   r!   r"   .test_distribution_strategy_on_subclassed_modelB  s    

*zQTestDistributionStrategyErrorCases.test_distribution_strategy_on_subclassed_modelc              	   C   s   |   tj }|tjjddd |tjjddd t rT|	d n6| 
td |	d W d    n1 s0    Y  W d    n1 s0    Y  d S )N   Zrelu)Z
activationr8   Zsoftmaxr/   r   )r>   r   models
Sequentialaddr   r   rE   rK   rA   rr   rs   )r   rO   rP   r!   r!   r"   7test_distribution_strategy_on_deferred_sequential_modelc  s    

zZTestDistributionStrategyErrorCases.test_distribution_strategy_on_deferred_sequential_modelc              	   C   sx   |  \ t }| td( tddg}||| W d    n1 sL0    Y  W d    n1 sj0    Y  d S )NzNPlease use `tf.keras.losses.Reduction.SUM` or `tf.keras.losses.Reduction.NONE`r$   r   )r>   r   ZMeanSquaredErrorrr   rs   r   Zasarray)r   rO   Zloss_objectyr!   r!   r"   +test_standalone_loss_without_loss_reduction  s    
zNTestDistributionStrategyErrorCases.test_standalone_loss_without_loss_reductionN)r*   r+   r,   rE   r\   rH   r]   r^   r_   combine"mirrored_strategy_with_gpu_and_cpurv   r{   r   Zone_device_strategyr   r   r?   'all_strategy_combinations_minus_defaultr   r!   r!   r!   r"   rb      s\   











+



	




	

rb   c                   @   sF   e Zd Zejjjejjjj	ejjjj
gddgejddd ZdS )'TestDistributionStrategyWithLossMaskingrc   rw   )rO   re   r3   c              
   C   s:  |    tjd tdgdggdgdggg}| ^ tj }|	tj
jddd |	tj
tj
jddd |jd| d	 W d    n1 s0    Y  tdgdggdgdggg}tjj||f}|d
}|d}|j|ddd}| |jd d d W d    n1 s,0    Y  d S )Ni9  r$   r   )r6   r$   )Z
mask_valueinput_shapeone)Zkernel_initializerr0   r4   r3   d   r~   r6   )ru   r9   r:   r4   )rp   r   r   seedarrayr>   r   r   r   r   r   ZMaskingZTimeDistributedr   rA   rE   dataDatasetfrom_tensor_slicesr   batchrC   assertEqualhistory)r   rO   r3   ru   rP   r   rQ   histr!   r!   r"   test_masking  s(    


.

z4TestDistributionStrategyWithLossMasking.test_maskingN)r*   r+   r,   rE   r\   rH   r]   r^   r_   r   r   r   &gradient_descent_optimizer_keras_v2_fnr   r!   r!   r!   r"   r     s   


	r   c                	   @   s   e Zd Zejjjejjj	e
 ejjjjddgejddd Zejjjejjj	e
 ejjjjejddd Zd	S )
.TestDistributionStrategyWithNormalizationLayerTF)fusedr3   c           
   	   C   sj  |   J | F tj }tjjdd|d}|| |jd| d W d    n1 s`0    Y  t	j
jdddd	}|d
}tjj||f}|d}t|d|}tjj|}|d}t|d|}|j|dddd |j|dd}	|	tj|j8 }	|	tj|j }	t	jj|	 ddd t	jj|	 ddd W d    n1 s\0    Y  d S )Nr~         皙?)r   momentumr   r0   r         @      $@i  r~   r   r   locscalesizefloat32r          r   r~   r9   r;   r:   r6   rU           皙?Zatol      ?rp   r>   r   r   r   r   ZBatchNormalizationr   rA   r   r   normalZastyperE   r   r   r   r   r?   Zbatch_wrapperrC   rY   backendevalbetagammatestingZassert_allclosemeanZstd)
r   rO   r   r3   rP   normru   rQ   predict_datasetoutr!   r!   r"   test_batchnorm_correctness  s4    


	
.


zITestDistributionStrategyWithNormalizationLayer.test_batchnorm_correctnessr3   c           	   	   C   sl  |   L | H tj }tjjddddd}|| |jd| d W d    n1 sb0    Y  t	j
jdd	d
d}|d}tjj||f}|d}t|d|}tjj|}|d}t|d|}|j|dddd |j|dd}|tj|j8 }|tj|j }t	jj| ddd t	jj| ddd W d    n1 s^0    Y  d S )Nr   r   FT)r   r   r   Zrenormr0   r   r   r   r   r   r   r   r   r   r   r~   r   r6   r   r   r   r   r   r   )	r   rO   r3   rP   r   ru   rQ   r   r   r!   r!   r"   &test_batchnorm_correctness_with_renorm  s6    	



.


zUTestDistributionStrategyWithNormalizationLayer.test_batchnorm_correctness_with_renormN)r*   r+   r,   rE   r\   rH   r]   r^   r_   r`   r?   ra   r   r   r   r   Ztpu_strategy_combinationsr   r!   r!   r!   r"   r     s&   


	
&


r   c                   @   s   e Zd Zejjjejjj	e
 ejjjjejddd Zejjjejjj	e
 ejjjjejddd ZdS )'TestDistributionStrategySaveLoadWeightsr   c              	   C   s   |    t|}|  t }|| d |j|ddd td}|	| t }|| d |
| |jt|dd |j|ddd W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr0   r$   r9   r:   z.h5r6   r   )rp   r?   rB   r>   r@   rA   rC   tempfilemktempsave_weightsload_weightsrY   rZ   r   rO   r3   rQ   rP   Zweights_fileZmodel_2r!   r!   r"   test_save_load_h5  s    	






z9TestDistributionStrategySaveLoadWeights.test_save_load_h5c              	   C   s  t |tjjjtjjjjjfr4|jjdkr4| 	d | 
  t|}|  t }|| d |j|ddd t }|| t }|| d || |jt|dd |j|ddd W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr$   z8MultiStep TPU Strategy deadlocks with optimizer restore.r0   r   r6   r   )rD   rE   rH   rI   rJ   rF   rG   rL   rM   ZskipTestrp   r?   rB   r>   r@   rA   rC   r   r   r   r   rY   rZ   r   r!   r!   r"   test_save_load_trackable4  s6    







z@TestDistributionStrategySaveLoadWeights.test_save_load_trackableN)r*   r+   r,   rE   r\   rH   r]   r^   r_   r`   r?   r   r   r   Zrmsprop_optimizer_keras_v2_fnr   r   r!   r!   r!   r"   r     s$   






r   c                   @   sP   e Zd Zejjjejjj	e
 dd Zejjje
 dd ZdS )"TestDistributionStrategyValidationc              
   C   s   |    | td tjjddd}tjjddd|}| N t||}t	j
jjd}d	}d
tj g}|j|||d W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S Nz,was not created in the distribution strategyr8   input)shapenamer   r   r   r|   r0   r1   r}   )rp   rr   rs   r   r   Inputr   r>   r   rE   rF   rG   r   r   r5   CategoricalAccuracyrA   r   rO   ru   r   rP   r3   r4   r5   r!   r!   r"   test_layer_outside_scopea  s    


z;TestDistributionStrategyValidation.test_layer_outside_scopec              
   C   s   |    | td tjjddd}tjjddd|}t||}| B t	j
jjd}d	}d
tj g}|j|||d W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S r   )rp   rr   rs   r   r   r   r   r   r>   rE   rF   rG   r   r   r5   r   rA   r   r!   r!   r"   test_model_outside_scopev  s    


z;TestDistributionStrategyValidation.test_model_outside_scopeN)r*   r+   r,   rE   r\   rH   r]   r^   r_   r`   r?   r   r   r   r!   r!   r!   r"   r   ^  s   



r   c                   @   sx   e Zd Zejjjejjjj	ejjjj
gddgddd Zejjjejjjj	ejjjj
gddgddd ZdS )	(TestDistributionStrategyWithStaticShapesrc   rw   rd   c              	   C   sj   |  N | td" tjjdddd W d    n1 s>0    Y  W d    n1 s\0    Y  d S )NzQThe `batch_size` argument \(5\) must be divisible by the number of replicas \(2\)r   r7   r   r   
batch_sizer   )r>   rr   rs   r   r   r   )r   rO   r!   r!   r"   3test_input_batch_size_not_divisible_by_num_replicas  s    	
z\TestDistributionStrategyWithStaticShapes.test_input_batch_size_not_divisible_by_num_replicasc                 C   s   t jdt jd}t jdt jd}tjj||f}|d}|jddd}|	 T t
jjddd	d
}t
jjddd|}t
||}|jdddgd W d    n1 s0    Y  |j|ddd |j|dd || d S )N)r~   r8   rx   )r~   r   r   r~   T)Zdrop_remainderr   r   r   r   r   r   r/   r0   r1   r2   r$   r7   r   r   )r   Zzerosr   rE   r   r   r   r   r   r>   r   r   r   r   r   rA   rC   rV   rY   )r   rO   r   targetsrQ   ru   r   rP   r!   r!   r"   test_static_input_batch_size  s    	

0zETestDistributionStrategyWithStaticShapes.test_static_input_batch_sizeN)r*   r+   r,   rE   r\   rH   r]   r^   r_   r   r   r   r   r!   r!   r!   r"   r     s    



	


r   __main__)#r-   r   r   numpyr   Ztensorflow.compat.v2rF   v2rE   Zabsl.testingr   r   r   Zkeras.distributer   r?   r   r   r=   ZCallbackr   r_   ZTestCaser.   rb   r   r   r   r   r   r*   r\   rH   Zmulti_process_runnerZ	test_mainr!   r!   r!   r"   <module>   sF   +

w

 O

&

`

H

,

.
