a
    +=ic                    @   s  d Z ddlZddlZddlm  mZ ddlm	Z	 ddl
Z
ddl
mZ ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m!Z! ddl m"Z" ddl#m$Z$ dZ%dZ&dZ'dZ(dd Z)e(fddZ*dd Z+dd  Z,dYd!d"Z-d#d$ Z.d%d& Z/d'd( Z0d)d* Z1dZd+d,Z2d-d. Z3d/d0 Z4d1d2 Z5d3d4 Z6d5d6 Z7d7d8 Z8d9d: Z9d;d< Z:d=d> Z;G d?d@ d@e
j<j=Z>G dAdB dBej?j@e	j@ZAG dCdD dDej?j@e	j@ZBG dEdF dFej?j@e	j@ZCG dGdH dHej?j@e	j@ZDeEdIG dJdK dKej?j@e	j@ZFdLdM ZGdNdO ZHdPdQ ZIG dRdS dSej?j@e	j@ZJG dTdU dUe
jKZLG dVdW dWej?j@e	j@ZMeNdXkreO  ejPjQjRS  dS )[z7Tests for tf.keras models using tf.distribute.Strategy.    N)parameterized)backend)distributed_training_utils)distributed_training_utils_v1)multi_worker_testing_utils)optimizer_combinations)all_strategies) multi_worker_mirrored_strategies)"strategies_minus_default_minus_tpu)strategies_minus_tpu)tpu_strategies)base_layer_utils)policy)gradient_descent)
test_utils)losses_utils)np_utils)SimpleClusterResolveri9     
      c                  C   sN   t j } | t jjddtd | t jd | t jjtdd | S )N   relu
activationinput_shape皙?softmaxr   )	kerasmodels
SequentialaddlayersDense_INPUT_SIZEDropout
_NUM_CLASSmodel r+   z/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/keras/distribute/distribute_strategy_test.pysimple_sequential_model@   s    
r-   c                 C   s   G dd dt j}|| S )Nc                       s$   e Zd Z fddZdd Z  ZS )z+simple_subclassed_model.<locals>._SimpleMLPc                    s   t    tj|| _d S N)super__init__r    r$   r%   dense)self
num_labels	__class__r+   r,   r0   L   s    
z4simple_subclassed_model.<locals>._SimpleMLP.__init__c                 S   s
   |  |S r.   )r1   r2   inputsr+   r+   r,   callP   s    z0simple_subclassed_model.<locals>._SimpleMLP.call)__name__
__module____qualname__r0   r8   __classcell__r+   r+   r4   r,   
_SimpleMLPK   s   r=   )r    Model)r3   r=   r+   r+   r,   simple_subclassed_modelJ   s    r?   c                  C   s|   t jjddd} t jjddd}t jj| |gdd}t jjddd	d
|}t jjdddd
|}t jj| |g||gd}|S )Nr   input_ashapenameinput_bmergerD      r   dense_2r   rD   r   Zdense_3r7   outputs)r    r$   InputZconcatenater%   r!   r>   )rA   rE   mergedoutput_coutput_dr*   r+   r+   r,   'simple_multi_inputs_multi_outputs_modelV   s    rQ   c                  C   s   t jtdddtd\\} }\}}t jtdddtd\\}}\}}t jtdddtd\\}}	\}
}	t|}t|}t|}t|}| ||||d}|||
||d}||fS )N2   r@   rH   )Ztrain_samplesZtest_samplesr   num_classesZrandom_seedr   )   )rA   rE   Zinput_mrO   rP   )r   Zget_test_data_TRAIN_SIZE_RANDOM_SEEDr   Zto_categorical)Za_trainZc_trainZa_testZc_testZb_trainZd_trainZb_testZd_testZm_train_Zm_testZ
train_dataZ	test_datar+   r+   r,   #get_multi_inputs_multi_outputs_datag   sL    



rX   c                 C   s4   |r|  |} t|r&| j|ddS | |S d S )NTZdrop_remainder)repeatr   is_tpu_strategybatch)dataset
batch_sizedistributionrZ   r+   r+   r,   batch_wrapper   s
    

r`   c                  C   s4   t jjddd} t jjddd| }t | |}|S )NrH   inputrB      r1   rG   r    r$   rM   r%   r>   xyr*   r+   r+   r,   	get_model   s    rh   c                  C   s8   t jjddd} t jjddddd| }t | |}|S )	N   rb   rB   rj   oneszerosr1   )kernel_initializerZbias_initializerrD   rd   re   r+   r+   r,   get_sample_weights_model   s    rn   c                 C   sL   t jdt jd}t jdt jd}tjj||f}|d}t|d| }|S )Nr   rH   Zdtyper   rc   d   r   	nprl   float32tfdataDatasetfrom_tensor_slicesrZ   r`   )r_   r7   targetsr]   r+   r+   r,   get_dataset   s    
r{   c                 C   s8   t jdt jd}tjj|}|d}t|d| }|S )Nro   rp   rr   r   rs   )r_   r7   r]   r+   r+   r,   get_predict_dataset   s
    
r|   c                 C   sL   |d ur| |f}dd }n| }dd }t jj|}||jddd}|S )Nc                 S   s   dS NTr+   )inptargetr+   r+   r,   <lambda>       zCconvert_numpy_to_dataset_with_unknown_cardinality.<locals>.<lambda>c                 S   s   dS r}   r+   r~   r+   r+   r,   r      r   r   TrY   )rv   rw   rx   ry   filterr\   )r7   rz   Zinput_slicesdummy_opZoriginal_datasetZds_with_unknown_cardinalityr+   r+   r,   1convert_numpy_to_dataset_with_unknown_cardinality   s    

r   c                  C   s~   t jjddd} t jjddd}t jjddd}t jjdd	d}|| }||}t jjd
dd|}t j| |g||g}|S )Nra   rA   rB   )   rE      dense_1rG   rI         ?Zdropout)r    r$   rM   r%   r'   r!   r>   )abr   rI   cder*   r+   r+   r,   multi_input_output_model   s    r   c                   C   s   t jjjjtddgdS Ngrapheagerr_   mode)rv   __internal__testcombinationscombiner   r+   r+   r+   r,   strategy_minus_tpu_combinations   s    
r   c                   C   s   t jjjjtddgdS r   rv   r   r   r   r   r   r+   r+   r+   r,   tpu_strategy_combinations   s    
r   c                   C   s   t jjjjtdgdS )Nr   r   r   r+   r+   r+   r,   $tpu_strategy_combinations_graph_only   s    
r   c                   C   s   t jjjjtdgdS )Nr   r   )rv   r   r   r   r   r	   r+   r+   r+   r,   -multi_worker_strategy_combinations_eager_only   s    
r   c                   C   s   t  t  t  S r.   )r   r   r   r+   r+   r+   r,   all_strategy_combinations   s    r   c                   C   s@   t jjjjt jjjjt jjjjt jjjjt jjjj	gddgdS r   )
rv   r   r   r   r   
distributeone_device_strategyone_device_strategy_gpu"mirrored_strategy_with_gpu_and_cpumirrored_strategy_with_two_gpusr+   r+   r+   r,   /all_strategy_minus_default_and_tpu_combinations   s    




r   c                   C   s   t  t  t  S r.   )r   r   r   r+   r+   r+   r,   'all_strategy_combinations_minus_default
  s    r   c                  C   s   t jjjt t jjjjtjtj	tj
tjtjtjtjtjtjtjtjtjgd} t jjjjtdgtjtj	tj
tjtjtjtjtjgd}t jjjjtdgtjtjtjtjgd}t jjjjtdgtjtjtjtjtjtjtjtjgd}| | | | S )N	optimizerr   )r_   r   r   r   )rv   r   r   r   timesr   r   r   Zadagrad_optimizer_v1_fnZadam_optimizer_v1_fnZ gradient_descent_optimizer_v1_fnZrmsprop_optimizer_v1_fnZadadelta_optimizer_keras_v2_fnZadagrad_optimizer_keras_v2_fnZadam_optimizer_keras_v2_fnZadamax_optimizer_keras_v2_fnZ&gradient_descent_optimizer_keras_v2_fnZnadam_optimizer_keras_v2_fnZrmsprop_optimizer_keras_v2_fnZftrl_optimizer_keras_v2_fnr   r	   )Znon_tpu_strategiesZtpu_strategies_graphZtpu_strategies_eagerZmulti_worker_eagerr+   r+   r,   #strategy_and_optimizer_combinations  sz    





r   c                       sX   e Zd Z fddZdddZdddZddd	Zdd
dZdddZdddZ	  Z
S )BatchCountingCBc                    s2   t    g | _g | _g | _g | _g | _g | _d S r.   )r/   r0   train_begin_batchestrain_end_batchestest_begin_batchestest_end_batchespredict_begin_batchespredict_end_batchesr2   r4   r+   r,   r0   U  s    
zBatchCountingCB.__init__Nc                 C   s   | j | d S r.   )r   appendr2   r\   Zlogsr+   r+   r,   on_train_batch_begin^  s    z$BatchCountingCB.on_train_batch_beginc                 C   s   | j | d S r.   )r   r   r   r+   r+   r,   on_train_batch_enda  s    z"BatchCountingCB.on_train_batch_endc                 C   s   | j | d S r.   )r   r   r   r+   r+   r,   on_test_batch_begind  s    z#BatchCountingCB.on_test_batch_beginc                 C   s   | j | d S r.   )r   r   r   r+   r+   r,   on_test_batch_endg  s    z!BatchCountingCB.on_test_batch_endc                 C   s   | j | d S r.   )r   r   r   r+   r+   r,   on_predict_batch_beginj  s    z&BatchCountingCB.on_predict_batch_beginc                 C   s   | j | d S r.   )r   r   r   r+   r+   r,   on_predict_batch_endm  s    z$BatchCountingCB.on_predict_batch_end)N)N)N)N)N)N)r9   r:   r;   r0   r   r   r   r   r   r   r<   r+   r+   r4   r,   r   T  s   	




r   c                   @   s(  e Zd Zejjje dd Z	ejjje dd Z
ejjje dd Zejjje dd Zejjje d	d
 Zejjje dd Zejjje dd Zejjjejjjjejjjjgddgddd Zejjje dd Zejjjejjjjeddgdejjjjedgd dd Zejjje dd Zejjjejjje ejjjjddgddd Zejjjejjje dd  Zejjje d!d" Zejjjejjje d#d$ Zejjje d%d& Zd'S )('TestDistributionStrategyWithNumpyArraysc                 C   s   d}t |s|j}|  t tj|dd d d\}}| |d|  | |d tj|dd d d\}}| |d|  | |d W d    n1 s0    Y  d S )N      ?@   stepsr^       r      rj   r   global_batch_size_supportednum_replicas_in_synccached_sessionr   get_input_paramsassertEqualr2   r_   Zreplica_scale_factorr   r^   r+   r+   r,   4test_calculating_input_params_no_steps_no_batch_sizet  s     


z\TestDistributionStrategyWithNumpyArrays.test_calculating_input_params_no_steps_no_batch_sizec              	   C   s^  d}t |s|j}|  * tj|ddd d\}}| |d|  | |d tj|ddd d\}}| |d|  | |d | td" tj|ddd d W d    n1 s0    Y  |dkrtj|dd	d d\}}| |d
 | |d	 n@| td" tj|ddd d W d    n1 s00    Y  W d    n1 sP0    Y  d S )Nr   r   rj   r   r   r   znot divisible by steps?   rH      z4could not be sharded evenly across the sync replicas)	r   r   r   r   r   r   r   assertRaisesRegex
ValueErrorr   r+   r+   r,   6test_calculating_input_params_with_steps_no_batch_size  sH    

$z^TestDistributionStrategyWithNumpyArrays.test_calculating_input_params_with_steps_no_batch_sizec                 C   s   d}t |s|j}|  t tj|dd dd\}}| |d | |d|  tj|dd dd\}}| |d | |d|  W d    n1 s0    Y  d S )Nr   r   r   r   rc   r   r   r   r   r+   r+   r,   6test_calculating_input_params_no_steps_with_batch_size  s     


z^TestDistributionStrategyWithNumpyArrays.test_calculating_input_params_no_steps_with_batch_sizec              	   C   s   |   | tj|dddd\}}| |d | |d | td" tj|dddd W d    n1 sl0    Y  W d    n1 s0    Y  d S )Nr   r   rH   r   zless than samples requiredr      )r   r   r   r   r   r   )r2   r_   r   r^   r+   r+   r,   8test_calculating_input_params_with_steps_with_batch_size  s    

z`TestDistributionStrategyWithNumpyArrays.test_calculating_input_params_with_steps_with_batch_sizec           	   
   C   s   |    |  tj}|d}t }d}dg}|j|||d tjdtjd}tjdtjd}|j	||dd	d
||fd |
|| |j
||dd || |j|dd W d    n1 s0    Y  W d    n1 s0    Y  d S )NMbP?msemaemetricsr   rH   rp   r   rc   rj   r   r   epochsr^   verbosevalidation_datarT   r^   )r   scopegradient_descent_kerasSGDrh   compilert   rl   ru   fitevaluatepredict)	r2   r_   optimizer_fnr   r*   lossr   r7   rz   r+   r+   r,   $test_calling_model_with_numpy_arrays  s,    


zLTestDistributionStrategyWithNumpyArrays.test_calling_model_with_numpy_arraysc                 C   s  t |tjjjjjtjjjtjjjtjjjjjfr<| d t	
|rLd}nd}|  > |  t| tj}|d}tjjddd}tjjdd	d
|}tjjddd|}t||}d}dg}	|j|||	d tjddd}
tjddd}|j|
|ddd|
|fd ||
| |j|
|dd ||
 |j|
dd W d    n1 s`0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nb/152097775mixed_bfloat16mixed_float16r   ra   rb   rB   rc   r1   rG   r   ru   rp   r   r   r   r   r   rj   r   r   r   rT   r   )
isinstancerv   compatv1r   experimentalParameterServerStrategyCentralStorageStrategyskipTestr   r[   r   r   r   Zpolicy_scoper   r   r    r$   rM   r%   Z
Activationr>   r   rt   rl   r   r   r   )r2   r_   policy_namer   r   rf   rg   r*   r   r   r7   rz   r+   r+   r,   'test_calling_model_with_mixed_precision$  sL    	

	
zOTestDistributionStrategyWithNumpyArrays.test_calling_model_with_mixed_precisionc              	      s  t |tjjjjjtjjjtjjjtjjjjjfr<| d t	
|rLd}nd}G dd dtjj}|   | Z ||d  fdd}t rt|}||\}}| | | | W d    n1 s0    Y  W d    n1 s0    Y  d S )	Nr   r   r   c                   @   s   e Zd Zdd Zdd ZdS )z_TestDistributionStrategyWithNumpyArrays.test_operator_overload_mixed_precision.<locals>.MyLayerc                 S   s    |  dd| _|  dd| _d S )Nvr+   )
add_weightr   v2r2   rW   r+   r+   r,   buildq  s    zeTestDistributionStrategyWithNumpyArrays.test_operator_overload_mixed_precision.<locals>.MyLayer.buildc                 S   s   || j 7 }| j| S r.   r   r   )r2   r~   r+   r+   r,   r8   u  s    
zdTestDistributionStrategyWithNumpyArrays.test_operator_overload_mixed_precision.<locals>.MyLayer.callNr9   r:   r;   r   r8   r+   r+   r+   r,   MyLayerp  s   r   rp   c                     s\   t dg} t } | }W d    n1 s20    Y  || j jg\}}||fS Nr   )rt   arrayrv   GradientTapegradientr   r   )rf   taperg   grad_v1grad_v2layerr+   r,   run_fn|  s
    
&z^TestDistributionStrategyWithNumpyArrays.test_operator_overload_mixed_precision.<locals>.run_fn)r   rv   r   r   r   r   r   r   r   r   r[   r    r$   Layerr   r   executing_eagerlyfunctionrunZassertIsNotNone)r2   r_   r   r   r  r   r  r+   r  r,   &test_operator_overload_mixed_precisionX  s*    	

	


zNTestDistributionStrategyWithNumpyArrays.test_operator_overload_mixed_precisionr   r   r   c              
   C   s   |    |  tjtjdg}tdgg}t	
 }||}W d    n1 s^0    Y  |||j}t }| td" |t||j W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nrj   r   z)cannot be called in cross-replica context)r   r   r    r!   r"   r$   r%   rt   r   rv   r   r   trainable_variablesr   r   r   RuntimeErrorapply_gradientszip)r2   r_   r*   rf   r   rg   Z	gradientsr   r+   r+   r,   4test_optimizer_in_cross_replica_context_raises_error  s    
&
z\TestDistributionStrategyWithNumpyArrays.test_optimizer_in_cross_replica_context_raises_errorc              	   C   s6  |    | 6 tj}|dd}t }d}||| W d    n1 sP0    Y  tjtjdtj	d}tjtjdtj	d}||g}tjtjdtj	d}	tjtjdtj	d}
|	|
g}|j
||dd	d
d ||| |j||d	d || |j|d	d W d    n1 s(0    Y  d S )Nr   Zlearning_rater   r   rp   )r   r   )r   r   rj   rT   r   )r   r^   r   r   )r   r   r   r   r   r   rt   asarrayrandomru   r   r   r   )r2   r_   r   r   r*   r   
input_a_np
input_b_npr7   output_d_npoutput_e_nprz   r+   r+   r,   +test_calling_model_with_nested_numpy_arrays  s,    

*
zSTestDistributionStrategyWithNumpyArrays.test_calling_model_with_nested_numpy_arraysc           	   	   C   s  |    |  t }tjjjjdd}d}||| t	
dgdgdgdggt	j}t	
dgdgd	gd
ggt	j}t	
g dt	j}|j||d|dd}| |d |j||ddd}| |d W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   r  r   r   rj   r   rH   rc      rT   g      ?r   g      ?rj   )r^   Zsample_weightr        @%@)r^   r         +@)r   r   rn   rv   r   r   trainRMSPropOptimizerr   rt   r   ru   r   assertAllClose)	r2   r_   r*   r   r   r7   rz   sample_weightsresultr+   r+   r,   test_numpy_with_sample_weights  s$    	zFTestDistributionStrategyWithNumpyArrays.test_numpy_with_sample_weightsc           
   	   C   s   |    | 6 t }tj}|dd}d}||| W d    n1 sN0    Y  tjtjdtj	d}tjtjdtj	d}||g}|
|}	| |	d | dd	g|	d
 j | dd	g|	d j W d    n1 s0    Y  d S )Nr   r  r   )r  rH   rp   )r  r   r   r  r   r   rj   )r   r   r   r   r   r   rt   r  r  ru   r   	assertLenassertAllEqualrC   )
r2   r_   r*   r   r   r   r  r  r7   Zoutsr+   r+   r,   test_flatten_predict_outputs  s    


*
zDTestDistributionStrategyWithNumpyArrays.test_flatten_predict_outputsrc   r  r   c              	   C   sP  |   0 tjjjd}d}dtj g}|	 & t
 }|j|||d W d    n1 sb0    Y  t
 }|j|||d tjdd}tjdd}	||  |||	}
td| }| j|j||	||d	d
d  |
d
d  ddd | j|j||	|dd
d  |
d
d  ddd W d    n1 sB0    Y  d S )Nr   r   r   r   ro   ru   rq         $@r^   r   rj   h㈵>ZatolZrtolr   )r   rv   r   r   r  GradientDescentOptimizerr    r   CategoricalAccuracyr   rh   r   rt   r  astypeset_weightsget_weightsr   ceilr  )r2   r_   r^   r   r   r   model_with_ds_strategy	cpu_modelrf   rg   Zevaluate_ground_truthr   r+   r+   r,    test_evaluate_with_partial_batch  s@    
.


zHTestDistributionStrategyWithNumpyArrays.test_evaluate_with_partial_batchc              	   C   s   |    tjjjd}d}| " t }||| W d    n1 sN0    Y  t }||| t	j

dt	j}||  ||}| j|j|ddd|ddd | j|j|dd	|ddd W d    n1 s0    Y  d S )
Nr   r   ro   rc   rH   r%  r&  r'  r   )r   rv   r   r   r  r(  r   rh   r   rt   r  r*  ru   r+  r,  r   r  )r2   r_   r   r   r.  r/  r7   Zpredict_ground_truthr+   r+   r,   test_predict_with_partial_batchI  s.    

*
zGTestDistributionStrategyWithNumpyArrays.test_predict_with_partial_batchc              	   C   s   |    tjjjd}G dd dtjj}|	  tj
 }|tjjddtd ||  |tjjtdd || td	tj}|j|d
dd |j|d
d |j|d
d W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   c                   @   s   e Zd ZdddZdS )zMTestDistributionStrategyWithNumpyArrays.test_no_target_model.<locals>.MyLayerNc                 S   s   | j t|dd |S )NT)r7   )add_lossrv   
reduce_sum)r2   r7   trainingr+   r+   r,   r8   y  s    zRTestDistributionStrategyWithNumpyArrays.test_no_target_model.<locals>.MyLayer.call)N)r9   r:   r;   r8   r+   r+   r+   r,   r   x  s   r   r   r   r   r   r   r   r   rj   r   r   steps_per_epochr   )r   rv   r   r   r  r(  r    r$   r  r   r!   r"   r#   r%   r&   r(   r   rt   rl   ru   r   r   r   )r2   r_   r   r   r*   r7   r+   r+   r,   test_no_target_modelq  s"    



z<TestDistributionStrategyWithNumpyArrays.test_no_target_modelc           	   	   C   s   |    tjjjd}d}| " t }||| W d    n1 sN0    Y  t }||| t	 \}}|d |d d}|
|  | j|j|ddd||d	d	d
 W d    n1 s0    Y  d S )Nr   r   rA   rE   rA   rE         r%  -C6?r'  )r   rv   r   r   r  r(  r   rQ   r   rX   r+  r,  r  r   )	r2   r_   r   r   r.  r/  
input_datarW   
input_dictr+   r+   r,   2test_predict_multi_output_model_with_partial_batch  s,    

*
	zZTestDistributionStrategyWithNumpyArrays.test_predict_multi_output_model_with_partial_batchc                 C   s   t  s| d G dd dtjj}| 0 t|dddg}|dd W d    n1 sd0    Y  t	j
jd	d
}t	j
jd	d
}| d }||| | d }| || d S )Nz.None gradients are not supported in graph modec                       s   e Zd Z fddZ  ZS )z]TestDistributionStrategyWithNumpyArrays.test_gradients_are_none.<locals>.DenseWithExtraWeightc                    s4   | j dddd| _t | | j dddd| _d S )Nextra_weight_1r+   rk   )rC   initializerextra_weight_2)r   rA  r/   r   rC  r2   r   r4   r+   r,   r     s    zcTestDistributionStrategyWithNumpyArrays.test_gradients_are_none.<locals>.DenseWithExtraWeight.build)r9   r:   r;   r   r<   r+   r+   r4   r,   DenseWithExtraWeight  s   rE  rc   )rc   )r   adamr   r   sizerj   )rv   r  r   r    r$   r%   r   r"   r   rt   r  normalr,  r   ZassertNotAllEqual)r2   r_   rE  r*   r7   rz   Z
old_kernelZ
new_kernelr+   r+   r,   test_gradients_are_none  s    

*z?TestDistributionStrategyWithNumpyArrays.test_gradients_are_noneN) r9   r:   r;   rv   r   r   r   generater   r   r   r   r   r   r   r	  r   r   r   r  r  r   r	   r   r#  r   r   r0  r1  r9  r@  rJ  r+   r+   r+   r,   r   q  s   



8





 

1

/





 



(




.


#




"
r   c                   @   s  e Zd Zejjje dd Z	ejjje dd Z
ejjje dd Zejjje dd Zejjje d	d
 Zejjje dd Zejjje dd Zejjje dd Zejjje dd Zejjje dd Zejjjejjjjejjjjejjjjgddgddd Zejjjejjjjejjjjgddgddd Zejjjejjjjejjjjejjjjgddgddd Zejjje dd Zejjjejjje ejjjjd d!gd"d#d$ Zejjjejjje d%d& Z ejjjejjje d'd( Z!ejjje" d)d* Z#ejjjejjjje$ddgdejjjje%dgd d+d, Z&d-S ).$TestDistributionStrategyWithDatasetsc              	   C   s   |    | F tj}|d}t }d}dtj g}|j|||d W d    n1 s^0    Y  t	|}|j
|ddd|dd |j
|ddd|dd |jt|dd	 W d    n1 s0    Y  d S )
Nr   r   r   r   rj   r   r   )r   r7  r   r   validation_stepsr8  )r   r   r   r   rh   r    r   r)  r   r{   r   r   r|   r2   r_   r   r   r*   r   r   r]   r+   r+   r,   "test_calling_model_on_same_dataset  s4    

.zGTestDistributionStrategyWithDatasets.test_calling_model_on_same_datasetc           
   
   C   sx  |   X | l tj}t }|j|dddtj gd t }|	|
  |j|dddtj gd W d    n1 s0    Y  t|}|j|ddd|ddd}g }tdD ],}|j|ddddd	 ||j|dd
 q| |jd dd |D  |jd}	|	s"|jd }	| |	dd |D  | |jd dd |D  W d    n1 sj0    Y  d S )Nr   r   r   )r   r   r   rj   F)r   r7  r   r   rM  shuffle)r   r7  r   rP  r8  Zval_lossc                 S   s   g | ]}|d  qS )r   r+   .0rf   r+   r+   r,   
<listcomp>0  r   zhTestDistributionStrategyWithDatasets.test_model_interleaved_eval_same_as_direct_eval.<locals>.<listcomp>val_mean_absolute_errorZval_maec                 S   s   g | ]}|d  qS ri   r+   rQ  r+   r+   r,   rS  9  r   Zval_categorical_accuracyc                 S   s   g | ]}|d  qS )r   r+   rQ  r+   r+   r,   rS  =  r   )r   r   r   r   rh   r   r    r   r)  r+  r,  r{   r   ranger   r   r   historyget)
r2   r_   r   Zuser_controlled_modelZinterleaved_modelr]   Zinterleaved_outputZuser_controlled_outputrW   rT  r+   r+   r,   /test_model_interleaved_eval_same_as_direct_eval  sp    
$
zTTestDistributionStrategyWithDatasets.test_model_interleaved_eval_same_as_direct_evalc              	   C   sZ  |   : | H tj}|dd}t }d}dtj g}|j|||d W d    n1 sb0    Y  t	j

dd}t	j

dd}t	j

d	d}	t	j

d	d}
tjj||f|	|
ff}|d
}|d}|j|dddd tjj||d|	|
ff}|d
}|d}|j|dddd W d    n1 sL0    Y  d S )Nr   r  r   r   r   ro   ru   r   r   )r   r   rr   r   rj   r   r   r7  r   r:  )r   r   r   r   r   r    r   r)  r   rt   r  r*  rv   rw   rx   ry   rZ   r\   r   )r2   r_   r   r   r*   r   r   r  r  r  r  Zdataset_tupleZdataset_dictr+   r+   r,   +test_fit_with_tuple_and_dict_dataset_inputs@  s4    

.



zPTestDistributionStrategyWithDatasets.test_fit_with_tuple_and_dict_dataset_inputsc              	      s  t |r| d dd  |  n |  tjjg ddd}tjjg ddd}tjjd	d	gd
d}tjjdddgdd|}tjj	 fdddd}||||g}tj
j|||g||gd}||dj |jdd W d    n1 s0    Y  t rdd }	ndd }	tjg dtjd}
tjg dtjd}tjg dtjd}tjj|
||f|	d}|| W d    n1 s0    Y  d S )Nzb/142805125c                 S   s   t j|| }t|| S r.   )r    lossesZbinary_crossentropyrv   reduce_mean)r   labelweightZbcer+   r+   r,   custom_losso  s    zlTestDistributionStrategyWithDatasets.test_fit_with_dictionary_in_the_dataset_b135161171.<locals>.custom_loss)r   r   rH   imgrG   )r   r   rj   lblr   r_  r   rj   Zsamepaddingc                    s    |  S r.   r+   rf   r`  r+   r,   r   |  r   ziTestDistributionStrategyWithDatasets.test_fit_with_dictionary_in_the_dataset_b135161171.<locals>.<lambda>my_lossrK   rF  r   c                 S   s   | ||d}|fS N)ra  rb  r_  r+   ra  rb  r_  r7   r+   r+   r,   map_fn  s    zgTestDistributionStrategyWithDatasets.test_fit_with_dictionary_in_the_dataset_b135161171.<locals>.map_fnc                 S   s   | ||d}|i fS rh  r+   ri  r+   r+   r,   rj    s    )rR   r   r   rH   rp   )rR   r   r   rj   )rR   r   r   r   )r   r[   r   r   r   r    r$   rM   Conv2DLambdar!   r>   r2  	get_layeroutputr   rv   r  rt   rk   ru   rw   rx   ry   mapr\   r   )r2   r_   Z	input_imgZ	input_lblZinput_weightr   Zloss_lambdarg  r*   rj  Z	fake_imgsZ	fake_lblsZfake_weightsrw   r+   rf  r,   2test_fit_with_dictionary_in_the_dataset_b135161171g  sH    


*

zWTestDistributionStrategyWithDatasets.test_fit_with_dictionary_in_the_dataset_b135161171c              	   C   sv  |   V | F tj}|d}t }d}dtj g}|j|||d W d    n1 s`0    Y  t	j
dt	jd}t	j
dt	jd}|j||dd	d
j}	|j||d	d}
|j|d	d}tjj||f}|jd	dd}|j|ddj}||}tjj|}|jd	dd}||}| j|	|ddd | j|
|ddd | j||ddd W d    n1 sh0    Y  d S )Nr   r   r   r     rH   rp   rr  rc   rj   r   r   r^   r   TrY   r   r=  r'  )r   r   r   r   rh   r    r   r)  r   rt   rl   ru   r   rV  r   r   rv   rw   rx   ry   r\   r  )r2   r_   r   r   r*   r   r   r7   rz   fit_with_numpyeval_with_numpypredict_with_numpyr]   fit_with_dseval_with_dspredict_datasetpredict_with_dsr+   r+   r,   :test_fit_eval_and_predict_methods_on_dataset_without_steps  s>    
.

z_TestDistributionStrategyWithDatasets.test_fit_eval_and_predict_methods_on_dataset_without_stepsc              	   C   s  |dkrt |r| d |   | F tj}|d}t }d}dtj	
 g}|j|||d W d    n1 sz0    Y  tjdtjd}|j|d	d
}	t|}
| tj tjj|
tjjj ||
}| j|	|ddd W d    n1 s0    Y  d S )Nr   3partial batch not supported with TPU in graph mode.r   r   r   r   r   rH   rp   r   r   r=  r'  )r   r[   r   r   r   r   r   rh   r    r   r)  r   rt   rl   ru   r   r   r   	get_valuerv   rw   r   cardinalityUNKNOWN_CARDINALITYr  )r2   r_   r   r   r   r*   r   r   r7   rx  r{  r|  r+   r+   r,   >test_predict_on_dataset_with_unknown_cardinality_without_steps  s2    


.
zcTestDistributionStrategyWithDatasets.test_predict_on_dataset_with_unknown_cardinality_without_stepsc              	   C   s  |dkrt |r| d |dkr8t |r8| d |   | F tj}|d}t }d}dtj	
 g}|j|||d W d    n1 s0    Y  tjd	tjd
}tjdtjd
}	|j||	dddj}
|j||	dddj}|j||	dd}|j|dd}t||	}t|}| tj tjj|tjjj | tj tjj|tjjj ||}||}| j||ddd | j||ddd |j|ddj}|j|ddj}| j|
|ddd | j||ddd W d    n1 s0    Y  d S )Nr   z'caused segfault with TPU in eager mode.r   r~  r   r   r   r   )rr   rH   rp   )rr   rc   rj   r   rt  r   r   r=  r'  ru  )r   r[   r   r   r   r   r   rh   r    r   r)  r   rt   rl   ru   r   rV  r   r   r   r   r  rv   rw   r   r  r  r  )r2   r_   r   r   r   r*   r   r   r7   rz   rv  Zfit_with_numpy_multiple_epochsrw  rx  r]   r{  rz  r|  ry  Zfit_with_ds_multiple_epochsr+   r+   r,   6test_on_dataset_with_unknown_cardinality_without_steps  sv    


.

z[TestDistributionStrategyWithDatasets.test_on_dataset_with_unknown_cardinality_without_stepsc              	   C   s  |    | D t }d}dtj g}|jtjj	j
d||d W d    n1 s^0    Y  tjdtjd}tjdtjd}|j||dd	}|j|dd	}t||}	t|}
| tjtjj|	tjjj | tjtjj|
tjjj |j|	d
d}|j|
d
d}| j||ddd | j||ddd | td |j|	dd W d    n1 sr0    Y  W d    n1 s0    Y  d S )Nr   r   r   r   rq  rp   rs  r   r   rr   r8  r=  r'  z%Number of steps could not be inferredrj   ru  )r   r   rh   r    r   r)  r   rv   r   r   r  r(  rt   rl   ru   r   r   r   r   r   r  rw   r   r  r  r  r   r   r   )r2   r_   r*   r   r   r7   rz   rw  rx  r]   r{  rz  r|  r+   r+   r,   (test_on_dataset_with_unknown_cardinality?  sX    
$zMTestDistributionStrategyWithDatasets.test_on_dataset_with_unknown_cardinalityc              	   C   s   |    | F tj}|d}t }d}dtj g}|j|||d W d    n1 s^0    Y  t	|}|j
|dddd |j|ddd |jt|dd	 W d    n1 s0    Y  d S )
Nr   r   r   r   rj   r   rZ  r   r   r8  )r   r   r   r   rh   r    r   r)  r   r{   r   r   r   r|   rN  r+   r+   r,   ,test_fit_eval_and_predict_methods_on_datasetw  s    

.zQTestDistributionStrategyWithDatasets.test_fit_eval_and_predict_methods_on_datasetc              	   C   s   |    | ( t }d}|| | W d    n1 s@0    Y  t|}|j|dddd |j|ddd |jt|dd W d    n1 s0    Y  d S )Nr   rj   r   rZ  r  r8  )	r   r   rh   r   r{   r   r   r   r|   )r2   r_   r   r*   r   r]   r+   r+   r,   (test_fit_eval_and_predict_with_optimizer  s    

,zMTestDistributionStrategyWithDatasets.test_fit_eval_and_predict_with_optimizerr   r   r   c           
   	   C   s  |dkr|  d |   | 6 tj}|dd}t }d}||| W d    n1 s`0    Y  tjdtj	d}tjdtj	d}t
jj||f}	|	d	}	|	d
}	| td" |j|	dddd W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   zYTODO(b/120943676, b/120957836): Re-enable for graph once the validation code is restored.r   r  r   rY  rp   rq   rr   r   zis incompatible withrj   r   r   rZ  )r   r   r   r   r   rh   r   rt   rl   ru   rv   rw   rx   ry   rZ   r\   r   r   r   )
r2   r_   r   r   r   r*   r   r7   rz   r]   r+   r+   r,   test_dataset_wrong_input_shape  s$    



*

zCTestDistributionStrategyWithDatasets.test_dataset_wrong_input_shapec           	   	   C   s   |    | 6 tj}|dd}t }d}||| W d    n1 sN0    Y  tjdtjd}tjdtjd}t	j
j||f}|d}|j|dd	dd
 W d    n1 s0    Y  d S )Nr   r  r   )rr   r   rH   rp   )rr   r   rc   rr   rj   r   rZ  )r   r   r   r   rh   r   rt   rl   ru   rv   rw   rx   ry   rZ   r   )	r2   r_   r   r   r*   r   r7   rz   r]   r+   r+   r,   ,test_dataset_external_batch_input_validation  s    	


*
zQTestDistributionStrategyWithDatasets.test_dataset_external_batch_input_validationc              	   C   s  |    |  tjjddd}tjjddd|}tjd|}t||}| }t	j
}|d}d	}	d
g}
|j||	|
d W d    n1 s0    Y  d}t|tjjtjjjjfrd|j }tjdtjd}tjdtjd}tjj||f}| |}|j|dddd}| |jd
 d dd |  || W d    n1 s`0    Y  tjdtjd}tjj|}| |}|j|dd}tjdtjd}|  ||d W d    n1 s0    Y  d S )Nri   rb   rB   rj   rk   rm   gH.?g{Gzt?r   accr   rT   r   rj   rp   r   rZ  r   r   r8  )   rj   r   )!r   r   r    r$   rM   r%   r'   r>   r,  r   r   r   r   rv   r   ZMirroredStrategyr   r   r   rt   rk   ru   rw   rx   ry   rZ   r\   r   ZassertAlmostEqualrV  r+  r   assertArrayNear)r2   r_   rf   rg   zr*   Zinitial_weightsr   r   r   r   r^   r7   rz   r]   histr{  rn  Z
ref_outputr+   r+   r,   test_learning_phase_value  sD    
.


*z>TestDistributionStrategyWithDatasets.test_learning_phase_valuec              	   C   s   |    | 0 t }td}d}||| W d    n1 sH0    Y  t|}dd }|j|dddtj	
|gd | d	tj|jj W d    n1 s0    Y  d S )
N{Gz?r   c                 S   s   dS )Nr   r+   )rW   r+   r+   r,   schedule  s    zQTestDistributionStrategyWithDatasets.testOptimizerWithCallbacks.<locals>.schedulerj   r   r   )r   r7  r   	callbacksr   )r   r   rh   r   r   r   r{   r   r    r  ZLearningRateSchedulerr  r   r  r   lr)r2   r_   r*   r   r   r]   r  r+   r+   r,   testOptimizerWithCallbacks  s$    


*z?TestDistributionStrategyWithDatasets.testOptimizerWithCallbacksrc   r  r   c              	   C   sf  |   F tjjjd}d}dtj g}|	 & t
 }|j|||d W d    n1 sb0    Y  t
 }|j|||d tjdd}tjdd}	tjj||	f}
||  |
|}td| }| j|j||d	d
d  |j||d	d
d  ddd | j||d
d  ||d
d  ddd W d    n1 sX0    Y  d S )Nr   r   r   r   ro   ru   rq   r$  r8  rj   r&  r'  )r   rv   r   r   r  r(  r    r   r)  r   rh   r   rt   r  r*  rw   rx   ry   r+  r,  r\   r-  r  r   )r2   r_   r^   r   r   r   r.  r/  rf   rg   r]   dataset_with_partial_batchr   r+   r+   r,   -test_evaluate_with_dataset_with_partial_batch-  s>    	
.
zRTestDistributionStrategyWithDatasets.test_evaluate_with_dataset_with_partial_batchc           	   	   C   s   |    tjjjd}d}| " t }||| W d    n1 sN0    Y  t }||| t	j

dt	j}tjj|}|d}||  | j|j|dd|j|ddddd W d    n1 s0    Y  d S )	Nr   r   ro   rc   rH   r8  r&  r'  )r   rv   r   r   r  r(  r   rh   r   rt   r  r*  ru   rw   rx   ry   r\   r+  r,  r  r   )	r2   r_   r   r   r.  r/  r7   r]   r  r+   r+   r,   ,test_predict_with_dataset_with_partial_batch]  s(    

*
zQTestDistributionStrategyWithDatasets.test_predict_with_dataset_with_partial_batchc              	   C   s   |    tjjjd}d}| " t }||| W d    n1 sN0    Y  t }||| t	 \}}|d |d d}tj
j|}	|	d}
||  | j|j|
dd|j|
ddd	d	d
 W d    n1 s0    Y  d S )Nr   r   rA   rE   r:  r;  r<  r8  r=  r'  )r   rv   r   r   r  r(  r   rQ   r   rX   rw   rx   ry   r\   r+  r,  r  r   )r2   r_   r   r   r.  r/  r>  rW   r?  r]   r  r+   r+   r,   ?test_predict_multi_output_model_with_dataset_with_partial_batch  s0    

*

zdTestDistributionStrategyWithDatasets.test_predict_multi_output_model_with_dataset_with_partial_batchc              	   C   s`  dd }t jddt jt jddt jd}t jdt jd}tjj	||f}|j
dd	d
}|   |  | \}}}tjj||g|d}	|	dd |	 }
|	j|dddj}| \}}}tjj||d|d}|dd ||
 |j|dddj}| j||ddd W d    n1 s20    Y  W d    n1 sR0    Y  d S )Nc                  S   sl   t jjddd} t jjddd}t jd| }t jd|}t j ||g}t jd|}| ||fS )Nr@   z_input_sorted_lastrB   )r   a_input_sorted_firstr   r   )r    r$   rM   r%   Add)rA   rE   Zintermediate_aZintermediate_brN   rn  r+   r+   r,   "_create_model_input_output_tensors  s    zTestDistributionStrategyWithDatasets.test_match_model_input_matches_with_dataset_tensors.<locals>._create_model_input_output_tensorsr   r   )r  r  )r   r   rp   rc   TrY   rK   sgdr   rj   r7  r   r=  r'  )rt   r  randr*  ru   rk   rv   rw   rx   ry   r\   r   r   r    r!   r>   r   r,  r   rV  r+  r  )r2   r_   r  r?  r   r]   rA   rE   rn  Zmodel_with_array_inputZmodel_weightsZmodel_with_array_input_fitZmodel_with_dict_inputZmodel_with_dict_input_fitr+   r+   r,   3test_match_model_input_matches_with_dataset_tensors  sH    


zXTestDistributionStrategyWithDatasets.test_match_model_input_matches_with_dataset_tensorsc           
   	   C   s6  |    |  t }tjjjjdd}d}||| t	
dgdgdgdggt	j}t	
dgdgd	gd
ggt	j}t	
g dt	j}tjj|||fd}|j|dd}	| |	d tjj||fd}|j|dd}	| |	d W d    n1 s0    Y  W d    n1 s(0    Y  d S )Nr   r  r   r   rj   r   rH   rc   r  rT   r  )r   r  r  )r   r   rn   rv   r   r   r  r  r   rt   r   ru   rw   rx   ry   r\   r   r  )
r2   r_   r*   r   r   r7   rz   r  dsr  r+   r+   r,    test_dataset_with_sample_weights  s$    	zETestDistributionStrategyWithDatasets.test_dataset_with_sample_weightsN)'r9   r:   r;   rv   r   r   r   rK  r   rO  rX  r[  rp  r}  r  r  r   r  r  r   r  r   r   r   r   r  r  r   r  r  r   r  r  r  r   r  r   r	   r  r+   r+   r+   r,   rL    s   



B

$

6

'

%

J

5







	








	
3




*





&

<


rL  c                       s6   e Zd Z fddZejjje	 dd Z
  ZS )(TestDistributionStrategyWithDatasetsFilec                    sh   t    tj|  d| _tjdtj	d}t
jj|}|t
jj}t
jj| j}|| d S )Nzinput.tfrecordr  rp   )r/   setUpospathjoinget_temp_dirinput_file_namert   rl   ru   rv   rw   rx   ry   ro  ioZserialize_tensorr   ZTFRecordWriterwrite)r2   r7   Zinput_datasetwriterr4   r+   r,   r    s    
z.TestDistributionStrategyWithDatasetsFile.setUpc           
      C   s   ~|  4 tj}|d}t }d}||| W d    n1 sD0    Y  tj| j}|	dd }dd }|
|jddd}tj }	tjjjj|	j_||	}|j|d	d
 d S )Nr   r   c                 S   s   t j| t jS r.   )rv   r  Zparse_tensorru   re  r+   r+   r,   r   :  r   z{TestDistributionStrategyWithDatasetsFile.test_predict_on_dataset_shard_options_file_multi_worker_mirrored.<locals>.<lambda>c                 S   s   dS r}   r+   r   r+   r+   r,   r   <  r   rT   TrY   rj   r8  )r   r   r   rh   r   rv   rw   ZTFRecordDatasetr  ro  r   r\   Optionsr   ZAutoShardPolicyFILEZexperimental_distributeZauto_shard_policyZwith_optionsr   )
r2   r_   r   r   r   r*   r   r]   r   optionsr+   r+   r,   @test_predict_on_dataset_shard_options_file_multi_worker_mirrored%  s     
*


ziTestDistributionStrategyWithDatasetsFile.test_predict_on_dataset_shard_options_file_multi_worker_mirrored)r9   r:   r;   r  rv   r   r   r   rK  r   r  r<   r+   r+   r4   r,   r    s
   
r  c                   @   sf   e Zd ZG dd dejjZG dd dejjZ	e
dd Zejjjejjje dd Zd	S )
TestRegularizerLossc                   @   s   e Zd Zdd ZdS )z'TestRegularizerLoss.IdentityRegularizerc                 C   s
   t |S r.   )rv   identityr2   rf   r+   r+   r,   __call__K  s    z0TestRegularizerLoss.IdentityRegularizer.__call__N)r9   r:   r;   r  r+   r+   r+   r,   IdentityRegularizerJ  s   r  c                   @   s   e Zd Zdd Zdd ZdS )zTestRegularizerLoss.AddLayerc                 C   s   | j dddt d| _d S )Nr   r+   rk   )rB  Zregularizer)r   r  r  r   r   r+   r+   r,   r   O  s    z"TestRegularizerLoss.AddLayer.buildc                 C   s
   || j  S r.   )r   r6   r+   r+   r,   r8   W  s    z!TestRegularizerLoss.AddLayer.callNr   r+   r+   r+   r,   AddLayerN  s   r  c                 C   s
   t |S r.   )rv   r]  )rW   Zy_predr+   r+   r,   loss_fnZ  s    zTestRegularizerLoss.loss_fnc                 C   s   d}t |s||j }|  tjjd|d}t |}tj	j
||d}td}|j|tjd |jtjdgdggtjdtjdgdggtjd|d | d	 }| d
| W d    n1 s0    Y  d S )Nr   ri   )rC   r^   rK   r   r   rp   rf   rg   r^   r         )r   r   r   r   r    r$   rM   r  r  r!   r>   r   r   r   r  r   rt   r   ru   r,  r   )r2   r_   r^   rf   rg   r*   optr   r+   r+   r,   test_regularizer_loss^  s$    


z)TestRegularizerLoss.test_regularizer_lossN)r9   r:   r;   r    ZregularizersZRegularizerr  r$   r  r  staticmethodr  rv   r   r   r   rK  r   r   r   r  r+   r+   r+   r,   r  I  s   


r  z$Uses Dense layers, which call matmulc                   @   s  e Zd Zejjje dd Z	ejjje dd Z
ejjjejjjjedgddd Zejjjejjjjedgdd	d
 Zejjjejjjjedgddd Zejjjejjjjedgddd Zejjjejjjjedgddd Zejjjejjjjedgddd Zejjjejjjjedgddd Zejjjejjjjedgddd Zejjjejjje dd Zejjjejjjjejjjjejjjjgddgejjejjejjgddd Z ejjjejjje dd Z!ejjje" dd  Z#ejjjejjje" d!d" Z$ejjjejjjjejjjj%ejjjj&ejjjjejjjjgdgdd#d$ Z'ejjjejjje" d%d& Z(ejjjejjjje)e* dgdd'd( Z+ejjjejjjje)e* dgdd)d* Z,ejjjejjjje-e. e* dgdd+d, Z/ejjjejjjjddgd-d.d/ Z0d0S )1'TestDistributionStrategyWithKerasModelsc                 C   s   |  R tj}|dd}t }d}||| tdtj}tdtj}W d    n1 s`0    Y  |j||ddd |j	|dd	 |j
||dd	 d S )
Nr   r  r   r5  )r   r   rj   r   rt  r   )r   r   r   r-   r   rt   rl   ru   r   r   r   r2   r_   r   r   r*   r   r7   rz   r+   r+   r,   .test_distribution_strategy_on_sequential_model  s    

,zVTestDistributionStrategyWithKerasModels.test_distribution_strategy_on_sequential_modelc                 C   s   |  V tj}|dd}t }d}||| tjdtjd}tjdtjd}W d    n1 sd0    Y  |j||dd |	| |
|| d S )	Nr   r  r   r   rp   r   rj   ru  )r   r   r   rh   r   rt   rl   ru   r   r   r   r  r+   r+   r,   .test_distribution_strategy_on_functional_model  s    

.
zVTestDistributionStrategyWithKerasModels.test_distribution_strategy_on_functional_modelr   r   c           	   	   C   s\  |  < G dd dtjj}ttjdg}|dd | }t	dt	d }}t
jj||f}|dd	}||}t
jj||f}|d
}||}|j|d|dd	|gd | |jd
 | |jd | |jd	 |jdkr8| td |j|d	d W d    n1 s.0    Y  W d    n1 sN0    Y  d S )Nc                   @   s2   e Zd Zdd Zd
ddZdddZddd	ZdS )zSTestDistributionStrategyWithKerasModels.test_distributed_dataset.<locals>.CBCounterc                 S   s   d| _ d| _d| _d S Nr   r   train_batchestest_batchesr   r+   r+   r,   r0     s    z\TestDistributionStrategyWithKerasModels.test_distributed_dataset.<locals>.CBCounter.__init__Nc                 S   s   |  j d7  _ d S Nrj   ru  r   r+   r+   r,   on_epoch_end  s    z`TestDistributionStrategyWithKerasModels.test_distributed_dataset.<locals>.CBCounter.on_epoch_endc                 S   s   |  j d7  _ d S r  r  r   r+   r+   r,   r     s    zfTestDistributionStrategyWithKerasModels.test_distributed_dataset.<locals>.CBCounter.on_train_batch_endc                 S   s   |  j d7  _ d S r  r  r   r+   r+   r,   r     s    zeTestDistributionStrategyWithKerasModels.test_distributed_dataset.<locals>.CBCounter.on_test_batch_end)N)N)Nr9   r:   r;   r0   r  r   r   r+   r+   r+   r,   	CBCounter  s   

r  rj   r  r   rr   r   rr   rj   r   r   r   r   r7  r   rM  r   r  %distributed dataset, you must specifyru  )r   r    r  Callbackr"   r$   r%   r   rt   rk   rv   rw   rx   ry   r\   rZ   experimental_distribute_datasetr   r   r  r  r   r   r   r   )	r2   r_   r  r*   
cb_counterrf   rg   r  val_dsr+   r+   r,   test_distributed_dataset  s8    


	z@TestDistributionStrategyWithKerasModels.test_distributed_datasetc              	   C   s  |   G dd dtjj}ttjdg}|dd | }dd }||}||}|j	|d|d	d
|gd | 
|jd | 
|jd | 
|jd
 |jdkr| td |j	|d
d W d    n1 s0    Y  W d    n1 s0    Y  d S )Nc                   @   s2   e Zd Zdd Zd
ddZdddZddd	ZdS )zbTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.CBCounterc                 S   s   d| _ d| _d| _d S r  r  r   r+   r+   r,   r0     s    zkTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.CBCounter.__init__Nc                 S   s   |  j d7  _ d S r  ru  r   r+   r+   r,   r    s    zoTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.CBCounter.on_epoch_endc                 S   s   |  j d7  _ d S r  r  r   r+   r+   r,   r     s    zuTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.CBCounter.on_train_batch_endc                 S   s   |  j d7  _ d S r  r  r   r+   r+   r,   r     s    ztTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.CBCounter.on_test_batch_end)N)N)Nr  r+   r+   r+   r,   r    s   

r  rj   r  r   c                 S   s:   t dt d }}tjj||f}|d }|S )Nr  r  r   )rt   rk   rv   rw   rx   ry   r\   rZ   )rW   rf   rg   r  r+   r+   r,   make_dataset   s    zeTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.make_datasetr   r   r   r  r   r  ru  )r   r    r  r  r"   r$   r%   r   Z!distribute_datasets_from_functionr   r   r  r  r   r   r   r   )r2   r_   r  r*   r  r  r  r  r+   r+   r,   'test_distributed_datasets_from_function  s4    

	
zOTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_functionc                 C   s`  t |tjjr| d | X td}tjj	ddd|}tj
 |}tjd|}t||}W d    n1 s~0    Y  |jdddd	 t }td
td }}|j||dd|gd | |jg d | |jg d |j||d|gd | |jg d | |jg d |j|d|gd | |jg d | |jg d d S )Nb/172032817)r   r   rH   rH   )kernel_sizerj   r  r   r   steps_per_execution)rr   r   r   rH   r  r   r^   r   r  )r   r   r      (   )	         '   1   r^   r  )r   rv   r   MultiWorkerMirroredStrategyr   r   r    rM   r$   rk  Flattenr%   r>   r   r   rt   rk   r   r   r   r   r   r   r   r   r   r   )r2   r_   r7   rf   rL   r*   bcrg   r+   r+   r,   test_host_training_loop  s(    


*z?TestDistributionStrategyWithKerasModels.test_host_training_loopc                 C   s>  t |tjjr| d | 6 td}tj	d|}t
||}W d    n1 s\0    Y  |jdddd t }tdtd	 }}|j||d
d|gd | |jg d | |jg d |j||d
|gd | |jg d | |jg d |j|d
|gd | |jg d | |jg d d S )Nr  r   rj   r  r   r   r  r  r  r   r  r   r   r  r  r  r  r  )r   rv   r   r  r   r   r    rM   r$   r%   r>   r   r   rt   rk   r   r   r   r   r   r   r   r   r   r   )r2   r_   r7   rL   r*   r  rf   rg   r+   r+   r,   .test_host_training_loop_last_partial_execution>  s$    


*zVTestDistributionStrategyWithKerasModels.test_host_training_loop_last_partial_executionc                 C   s  t |tjjr| d | 6 td}tj	d|}t
||}W d    n1 s\0    Y  |jdddd tdtd	 }}tjj||fd
}|dd }t }| td" |j|d
|gd W d    n1 s0    Y  |d
}	|j|	dd
|gd | |jg d | |jg d | td  |j||gd W d    n1 sl0    Y  |d
}
|j|
d|gd | |jg d | |jg d |d
}|j|d|gd | |jg d | |j g d d S )Nr  r   rj   r  r   r   r  r  r  r   c                  _   s   dS r}   r+   )argskwargsr+   r+   r,   r   l  r   zfTestDistributionStrategyWithKerasModels.test_host_training_loop_dataset_unknown_size.<locals>.<lambda>r  )r   r  rR   )r7  r   r  )r   r   r  r   r   r  )r  r  r  r  r  r  )r  )r   r  r  r  )!r   rv   r   r  r   r   r    rM   r$   r%   r>   r   rt   rk   rw   rx   ry   r\   r   r   r   r   r   rZ   r   r   r   r   r   r   r   r   r   )r2   r_   r7   rL   r*   rf   rg   r  r  Ztrain_dsZtest_dsZ
predict_dsr+   r+   r,   ,test_host_training_loop_dataset_unknown_size[  s6    


*0
0

zTTestDistributionStrategyWithKerasModels.test_host_training_loop_dataset_unknown_sizec                 C   sV  t |tjjr| d | 6 td}tj	d|}t
||}W d    n1 s\0    Y  |jdddd tdtd	 }}t }|j||d
d
|gd | |jddg | |jddg tdtd }}|j||d
|gd | |jdg | |jdg td}|j|d
|gd | |jdg | |jdg d S )Nr  r   rj   r  r   i  r  r  r  r   r  r   r  )rR   r   )rR   rj   r     )r   rv   r   r  r   r   r    rM   r$   r%   r>   r   rt   rk   r   r   r   r   r   r   r   r   r   r   r   )r2   r_   r7   rL   r*   rf   rg   r  r+   r+   r,   )test_host_training_loop_truncate_to_epoch  s(    


*
zQTestDistributionStrategyWithKerasModels.test_host_training_loop_truncate_to_epochc                 C   s  G dd dt jj}tdtd }}| 2 | }t |g}tjdddd}W d    n1 sj0    Y  |	|d t
|tjjjtjjjjjfr| td" |j||d	d
d W d    n1 s0    Y  n:|j||d	d
d | | |jd | | |jd d S )Nc                   @   s   e Zd Zdd Zdd ZdS )zOTestDistributionStrategyWithKerasModels.test_gradient_clipping.<locals>.MyLayerc                 S   s   t d| _t d| _d S r   rv   Variabler   r   r   r+   r+   r,   r     s    zUTestDistributionStrategyWithKerasModels.test_gradient_clipping.<locals>.MyLayer.buildc                 S   s   d| j  d| j  S )NrH   r   r  r+   r+   r,   r8     s    zTTestDistributionStrategyWithKerasModels.test_gradient_clipping.<locals>.MyLayer.callNr   r+   r+   r+   r,   r     s   r   r  r   g       @)ZclipnormZ	clipvaluer   znot supportedr   rj   r^   r   g      @r  )r    r$   r  rt   rk   r   r"   r   r   r   r   rv   r   r   r   r   r   r   r   r   r  r   r   )r2   r_   r   rf   rg   r  r*   r   r+   r+   r,   test_gradient_clipping  s(    
$2z>TestDistributionStrategyWithKerasModels.test_gradient_clippingc           	      C   s   t |tjjjtjjjjjfr(| d G dd dtj	j
}dd }tdtd }}| 2 | }t|g}tjd|gd}W d    n1 s0    Y  ||d	 |j||d
dd | | |jd | | |jd d S )Nz+Not supported with `CentralStorageStrategy`c                   @   s   e Zd Zdd Zdd ZdS )z\TestDistributionStrategyWithKerasModels.test_custom_gradient_transformation.<locals>.MyLayerc                 S   s   t d| _t d| _d S )Nr   r  r  r   r+   r+   r,   r     s    zbTestDistributionStrategyWithKerasModels.test_custom_gradient_transformation.<locals>.MyLayer.buildc                 S   s   || j  | j S r.   r   r  r+   r+   r,   r8     s    zaTestDistributionStrategyWithKerasModels.test_custom_gradient_transformation.<locals>.MyLayer.callNr   r+   r+   r+   r,   r     s   r   c                 S   s   dd | D S )Nc                 S   s   g | ]\}}t ||fqS r+   )rv   	ones_like)rR  gr   r+   r+   r,   rS    r   zyTestDistributionStrategyWithKerasModels.test_custom_gradient_transformation.<locals>.custom_transform.<locals>.<listcomp>r+   )Zgrads_and_varsr+   r+   r,   custom_transform  s    zeTestDistributionStrategyWithKerasModels.test_custom_gradient_transformation.<locals>.custom_transformr  r   )Zgradient_transformersr   r   rj   r          g       )r   rv   r   r   r   r   r   r   r    r$   r  rt   rk   r   r"   r   r   r   r   r  r   r   )	r2   r_   r   r  rf   rg   r  r*   r   r+   r+   r,   #test_custom_gradient_transformation  s(    

$zKTestDistributionStrategyWithKerasModels.test_custom_gradient_transformationc                 C   s   |   tjjdd}tjjddd|}tj|g|gd}|jddd	gd
 tjd	d}tjj
ddd}|j||ddd W d    n1 s0    Y  d S )Nr   rC   rH   r   r   rK   Zrmspropsparse_categorical_crossentropysparse_categorical_accuracyr   r   r   r   r   ru   r   rG  rj   r   r6  )r   r    r$   rM   r%   r>   r   rt   r  r*  randintr   )r2   r_   r~   outr*   rf   rg   r+   r+   r,   *test_distribution_strategy_one_dimensional  s    
zRTestDistributionStrategyWithKerasModels.test_distribution_strategy_one_dimensionalr   )r_   r   	reductionc                 C   s   t jt dd }t jd}t jd}tjj||f}|d}| }|j	dt
jj|dd |j|d	d
dd}| @ | }	|	j	dt
jj|dd |	j|d	d
dd}
W d    n1 s0    Y  | |jd |
jd d d S )Nc                  S   sV   t d} t jjddd| }t jjddd|}t jjddd|}t | |}|S Nr   r   rl   r  rj   r    rM   r$   r%   r>   r7   x1Zx2rL   r*   r+   r+   r,   
_get_model	  s    
zpTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_loss_reduction_types.<locals>._get_modelr   r   rj   r   r  )r  r  r   rj   F)r7  r   rP  r   r&  )rt   r  seedrV   rv   rw   rx   ry   r\   r   r    r\  MeanSquaredErrorr   r   r  rV  )r2   r_   r  r	  rf   rg   r]   r*   rV  ds_model
ds_historyr+   r+   r,   4test_distribution_strategy_with_loss_reduction_types	  s,    

$z\TestDistributionStrategyWithKerasModels.test_distribution_strategy_with_loss_reduction_typesc           	      C   s   dd }t dd}| }|d |j|dd}| . | }|d |j|dd}W d    n1 sr0    Y  | |j|j d S )Nc                  S   sv   t d} t jjddd| }t jjddd|}t jjddd|}t | |}|t| |t| |S r  )r    rM   r$   r%   r>   r2  rv   r]  r  r+   r+   r,   _make_model_with_add_lossA	  s    
z|TestDistributionStrategyWithKerasModels.test_distribution_strategy_with_symbolic_add_loss.<locals>._make_model_with_add_lossr   ru   r  rj   ru  )rt   rk   r*  r   r   r   r  rV  )	r2   r   r_   r  rf   r*   rV  r  r  r+   r+   r,   1test_distribution_strategy_with_symbolic_add_loss9	  s    



,zYTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_symbolic_add_lossc           	      C   s   dd }t dd}t dd}| }| |jd |dd |j||d	dd
}| B | }| |jd |dd |j||d	dd
}W d    n1 s0    Y  | |j	|j	 d S )Nc                     sl   t d} t jjddd| }t jjddd|}t jjddd  |}t | |}| fdd |S )Nr   r   rl   r  rj   c                      s   dt  j S )Ng      Y@)rv   r]  Zkernelr+   r   r+   r,   r   f	  r   zTestDistributionStrategyWithKerasModels.DISABLED_test_distribution_strategy_with_callable_add_loss.<locals>._make_model.<locals>.<lambda>)r    rM   r$   r%   r>   r2  r  r+   r  r,   _make_model_	  s    
zwTestDistributionStrategyWithKerasModels.DISABLED_test_distribution_strategy_with_callable_add_loss.<locals>._make_modelr   ru   r
  rj   r  r   r   r  )
rt   rk   r*  r!  r\  r   r   r   r  rV  )	r2   r_   r  rf   rg   r*   rV  r  r  r+   r+   r,   :DISABLED_test_distribution_strategy_with_callable_add_lossY	  s    

0zbTestDistributionStrategyWithKerasModels.DISABLED_test_distribution_strategy_with_callable_add_lossc           
         s
  G dd dt jj  fdd}tdd}tdd}| }| |jd |d	d
 |j	||||fddd}|
 f | }| |jd |d	d
 |j	||||fddd}t rdnd}	| |j|	 W d    n1 s0    Y  | |j|j d S )Nc                   @   s   e Zd Zdd Zdd ZdS )zhTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_in_call.<locals>.Biasc                 S   s   | j dddd| _d S )Nbiasrl   r+   rD   rB  rC   )r   r  rD  r+   r+   r,   r   	  s    znTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_in_call.<locals>.Bias.buildc                 S   s    | j t|ddd || j S )Nr  meanrD   aggregation)
add_metricrv   r]  r  r6   r+   r+   r,   r8   	  s    zmTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_in_call.<locals>.Bias.callNr   r+   r+   r+   r,   Bias	  s   r  c                     sL   t d} t jjddd| }  |}t jjddd|}t | |}|S r  r  r  r  r+   r,   _make_model_with_add_metric	  s    

zTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_in_call.<locals>._make_model_with_add_metricr   ru   r
  rj   r  r   r   r   rM  r   r    r$   r  rt   rk   r*  r!  r   r   r   r   rv   r  r  rV  
r2   r_   r  rf   rg   r*   rV  r  r  metrics_lenr+   r  r,   2test_distribution_strategy_with_add_metric_in_callz	  s(    
,zZTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_in_callc           
         s
  G dd dt jj  fdd}tdd}tdd}| }| |jd |d	d
 |j	||||fddd}|
 f | }| |jd |d	d
 |j	||||fddd}t rdnd}	| |j|	 W d    n1 s0    Y  | |j|j d S )Nc                   @   s   e Zd Zdd Zdd ZdS )zgTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_object.<locals>.Biasc                 S   s&   | j dddd| _tjjdd| _d S )Nr  rl   r+   r  r  rG   )r   r  r    r   ZMeanr  rD  r+   r+   r,   r   	  s    zmTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_object.<locals>.Bias.buildc                 S   s   |  | | || j S r.   )r  r  r  r6   r+   r+   r,   r8   	  s    zlTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_object.<locals>.Bias.callNr   r+   r+   r+   r,   r  	  s   r  c                     sL   t d} t jjddd| }  |}t jjddd|}t | |}|S r  r  r  r  r+   r,   "_make_model_with_add_metric_object	  s    

zTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_object.<locals>._make_model_with_add_metric_objectr   ru   r
  rj   r  r   r   r  r  )
r2   r_   r#  rf   rg   r*   rV  r  r  r!  r+   r  r,   1test_distribution_strategy_with_add_metric_object	  s(    
,zYTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_objectc           
      C   s   dd }t dd}t dd}| }| |jd |dd |j||||fd	d	d
}| f | }| |jd |dd |j||||fd	d	d
}t	 rd	nd}	| |j|	 W d    n1 s0    Y  | 
|j|j d S )Nc                  S   sX   t d} t jjddd| }t jjddd|}t | |}|jt|ddd |S )	Nr   r   rl   r  rj   Zmid_meanr  r  )r    rM   r$   r%   r>   r  rv   r]  )r7   r  rL   r*   r+   r+   r,   r  	  s    
zTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_outside_call.<locals>._make_model_with_add_metricr   ru   r
  rj   r  r   r   r  )rt   rk   r*  r!  r   r   r   r   rv   r  r  rV  r   r+   r+   r,   7test_distribution_strategy_with_add_metric_outside_call	  s&    	

,z_TestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_outside_callc           	      C   s   G dd dt jj}t | g}tg dg dg}|j|dd}tddgddgddgg}tg d	}tdd
g}| |j| | |j	| | |j
| d S )Nc                   @   s   e Zd ZdZdd ZdS )zTTestDistributionStrategyWithKerasModels.test_sparse_tensor_outputs.<locals>.ToSparsez5Create a sparse tensor based on a given dense tensor.c                 S   s<   t t |d}t ||}t j|dd}t j|||dS )Nr   int64)Zout_type)dense_shape)rv   where	not_equalZ	gather_ndrC   ZSparseTensor)r2   r7   indicesvaluesrC   r+   r+   r,   r8   
  s    zYTestDistributionStrategyWithKerasModels.test_sparse_tensor_outputs.<locals>.ToSparse.callN)r9   r:   r;   __doc__r8   r+   r+   r+   r,   ToSparse
  s   r-  rj   r   r   r   rH   r   r   r   r   rj   )rj   r   rH   rH   )r    r$   r  r"   rt   r   r   r"  r*  r+  r'  )	r2   r_   r-  r*   r>  rn  Zexpected_indicesexpected_valuesZexpected_dense_shaper+   r+   r,   test_sparse_tensor_outputs
  s    	zBTestDistributionStrategyWithKerasModels.test_sparse_tensor_outputsc                 C   sh   G dd dt jj}t |ddg}tg dg dg}|j|dd}d	gdd
gg}| || d S )Nc                       s*   e Zd ZdZd fdd	Zdd Z  ZS )zTTestDistributionStrategyWithKerasModels.test_ragged_tensor_outputs.<locals>.ToRaggedz5Create a ragged tensor based on a given dense tensor.rj   c                    s"   t  jf i | || _|| _d S r.   )r/   r0   _padding_ragged_rank)r2   rd  ragged_rankr  r4   r+   r,   r0   8
  s    z]TestDistributionStrategyWithKerasModels.test_ragged_tensor_outputs.<locals>.ToRagged.__init__c                 S   s   t jj|| j| jdS )N)rd  r4  )rv   ZRaggedTensorZfrom_tensorr2  r3  r6   r+   r+   r,   r8   =
  s    
zYTestDistributionStrategyWithKerasModels.test_ragged_tensor_outputs.<locals>.ToRagged.call)rj   )r9   r:   r;   r,  r0   r8   r<   r+   r+   r4   r,   ToRagged5
  s   r5  r   rc  r.  r/  r   r   rj   rH   )r    r$   r  r"   rt   r   r   r"  )r2   r_   r5  r*   r>  rn  r0  r+   r+   r,   test_ragged_tensor_outputs-
  s    zBTestDistributionStrategyWithKerasModels.test_ragged_tensor_outputsc                    s   d dd } fdd}  d | tdtj fdd}| }td	D ]}|D ]}|| qfq^W d    n1 s0    Y  d S )
Nr   c                  S   sj   t jjdd} t jjdd}t jjddd| }t jd|}t j| |g|}|t j|| |S )Nri   r  r   r   r   rj   )	r    r$   rM   r%   r!   r>   r2  r\  mean_squared_error)r7   labelsrf   rg   r*   r+   r+   r,   r	  V
  s    zhTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call.<locals>._get_modelc                     sJ   t jdd} d|  }| d} |d}tjj| |f}| }|S )Nr   rj   rH   ru   )	rt   r  r  r*  rv   rw   rx   ry   r\   )Zx_trainZy_trainr]   r   r+   r,   	_get_data_
  s    


zgTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call.<locals>._get_datag?c                    s6    fdd}j || fd}jtjjj|d dS )Nc                    s   t  }| }t j dd  tjdks8J t jd   }tj	| d |}t j
j| d}t jj|| W d    n1 s0    Y  ||j}t|j |S )Nc                 S   s   d S r.   r+   r  r+   r+   r,   r   t
  r   zTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call.<locals>.train_step.<locals>.step_fn.<locals>.<lambda>rj   r   )Zglobal_batch_size)rv   r   r   Zget_replica_contextZ
merge_calllenr\  r3  r    r7  nnZcompute_average_lossr   r   Zassert_equalr   r
  r  r  )r7   r   logitsZloss_from_modelr   Zgrads)r^   r*   r   r+   r,   step_fnn
  s(    

.
zyTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call.<locals>.train_step.<locals>.step_fn)r  )Zaxis)r  reducerv   r   ZReduceOpSUM)Zdist_inputsr=  Zper_replica_lossesr^   r_   r*   r   r+   r,   
train_stepl
  s    "zhTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call.<locals>.train_stepr   )r   r   r   rv   r  r  rU  )r2   r_   r	  r9  rA  r]   rW   rf   r+   r@  r,   ,test_correctness_of_add_loss_with_merge_callK
  s    			

*zTTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call)r   c              	   C   s   t jddd}ttj|ddddid}tjjjj	
|}| |tjjjj	j
 | td	` | 8 t }tjjjjd
d}d}||| W d    n1 s0    Y  W d    n1 s0    Y  d S )NrH   r   )Znum_workersZnum_psZworkerrj   ZGPUr   )cluster_specZ	task_typetask_idZnum_acceleratorszParameterServerStrategy*r   r  r   )r   Zcreate_in_process_clusterr   rv   r  ZClusterSpecr   r   r   r   r   ZassertIsInstancer   NotImplementedErrorr   r-   r  r   )r2   rC  Zcluster_resolverr_   r*   r   r   r+   r+   r,   ,test_unimplemented_parameter_server_strategy
  s6    


zTTestDistributionStrategyWithKerasModels.test_unimplemented_parameter_server_strategyN)1r9   r:   r;   rv   r   r   r   rK  r   r  r  r   r   r   r  r  r  r  r  r  r  r  r   r   r  r   r   r   ReductionV2ZAUTOSUM_OVER_BATCH_SIZEr?  r  r  r   r  r"  r   r   r$  r%  r   r	   r1  r6  r
   r   rB  rF  r+   r+   r+   r,   r    s  






3


5








$





!


$







$







-






,


%








I
r  c           
         s(  t j| dd}t jjdddd|}t jjdd|}t jjd	ddd|}t jjdd|}t jjd
d|}ttt	|d}t jj
dddd|}t jjdd dd|}t jj
|dd|}t j||d}	|	t j fdd| |	j|ddd |	||  |	j|ddd |	S )NimagesrG   r   r   r   r  r   r   Z	pool_sizer   	embedding   sparse_embeddingrJ   c                 S   s   t t | dS NrM  rv   r]  r3  re  r+   r+   r,   r   
  r   z6_functional_with_add_loss_and_metric.<locals>.<lambda>l1_lossr<  rK   c                    s   |   S r.   r+   re  l2r+   r,   r   
  r   r  l2_lossr  rD   )r    rM   r$   rk  MaxPooling2Dr  rv   r]  r3  squarer%   rl  r>   r2  r  )
r   rS   l1rT  r7   rf   rU  rR  rL   r*   r+   rS  r,   $_functional_with_add_loss_and_metric
  s(    rZ  c                    s  t t jjddd| dt jjddt jjddddt jjddt jjd	d
t jjddddt jj|dd
g}|dd} t	
t	|d }|| |jt j fdd|ddd |d	d}t jjfdddd
|}|| |j| ddd |S )Nr   r   r   r  r   r   r   rK  r   rJ  rL  rG   rN  rO  rJ   r<  rM  c                    s   t |  S r.   )rv   dividere  )rY  r+   r,   r   
  r   z6_sequential_with_add_loss_and_metric.<locals>.<lambda>r  rR  rV  c                    s    t t | |  d S rP  rQ  re  rS  r+   r,   r   
  r   rU  )r    r"   r$   rk  rW  r  r%   rm  Zget_output_atrv   r]  r3  r2  r  rl  )r   rS   rY  rT  r*   rf   rR  rU  r+   )rY  rT  r,   $_sequential_with_add_loss_and_metric
  s>    

r]  c                 C   s   t t jjddd| dt jjddt jjddddt jjddt j t jjd	dd
t jj|ddg}t j| dd}||}t j||d}|t	
|}|t	|}	t	t	t	|| d}
||
|  |j|
ddd t	t	t	||	 d}|||  |j|ddd |S )Nr   r   r   r[  r   rK  r   rJ  rN  r   r<  rG   rI  rK   rM  r  rU  rV  rR  )r    r"   r$   rk  rW  r  r%   rM   r>   rv   Z
zeros_liker  r]  r3  rX  r2  r  abs)r   rS   rY  rT  Z
base_modelr7   r<  r*   Zzero_logitsZ
one_logitsrU  rR  r+   r+   r,   _functional_with_layer_reuse
  s0    r_  c                
   @   sT   e Zd ZdZejjjejj	j
e ejj	jjeeegdgdgddd ZdS )9TestDistributionStrategyWithMultipleAddLossAndMetricCallsz=Tests complex models with multiple add loss and metric calls.r  r   )model_fnrY  rT  c           
      C   sz  t jt tjjt jjdd	t j
t jjddddf}|djd|j dd	}tjj|d jd
d  }| \ ||d||}|jtjjdtjjdtjjdtj tjjddgd W d    n1 s0    Y  tdD ]}| | qt!t"|j#|$|}	| %|	d dd | &|	d d | &|	d d | '|	d | |	d |  |	d  |	d d d S )N)r      rb  rj   rG  r   r   r   r   rT   TrY   rj   r=  )from_logitsr  )rd  r  r   r  g{Gz?r   rU  r  rR  r  r   gư>)(rt   r  r  rV   rv   rw   rx   ry   uniformr*  ru   r  rP  r\   r   r   Zget_structurerC   r   r   r    
optimizersadam_v2Adamr\  ZSparseCategoricalCrossentropyr   rG  rH  r   ZSparseCategoricalAccuracyrU  r   dictr  metrics_namesr   ZassertBetweenZassertGreaterZ
assertNear)
r2   r_   ra  rY  rT  r]   r   r*   unused_epochresultsr+   r+   r,   test_fit_and_evaluate!  sN    


$

zOTestDistributionStrategyWithMultipleAddLossAndMetricCalls.test_fit_and_evaluateN)r9   r:   r;   r,  rv   r   r   r   rK  r   r   r   r   rZ  r]  r_  rm  r+   r+   r+   r,   r`    s   


r`  c                       s2   e Zd ZdZ fddZdd Zd	ddZ  ZS )
DeterministicModelzDeterministic Model that always outputs the same initial result.

    It verifies the `call` method is run inside the same distribution
    strategy that the model was initially passed.
    c                    s   t    d | _|| _d S r.   )r/   r0   rf   strategy)r2   ro  r4   r+   r,   r0   f  s    
zDeterministicModel.__init__c                 C   s   t t jdd| _d S )Nr+   r  )rv   r  rk   rf   rD  r+   r+   r,   r   k  s    zDeterministicModel.buildNc                 C   s&   t j }|| jurtd| j| S )Nz0Model must execute call w/ the original strategy)rv   r   Zget_strategyro  r   rf   )r2   r7   r4  maskZactive_strategyr+   r+   r,   r8   n  s    

zDeterministicModel.call)NN)r9   r:   r;   r,  r0   r   r8   r<   r+   r+   r4   r,   rn  _  s   rn  c                   @   sl   e Zd ZdZejjjejj	jj
edgddd Zejjjejj	jj
ejjjjdgddd ZdS )	TestModelCapturesStrategyz0Tests that model creation captures the strategy.r   r   c           
      C   s  t jjt jddt jddf}|d|j }| & t|}t	j
jd}W d    n1 sf0    Y  |j|t	j dgd | |jj d tdD ]}|| q||}tt|j|}| |d d	 | |d
 d t jdd |jD }|D ]}| |j | q
|jj!D ]}| |j | q,|j"D ]}| |j | qL|  t	j# }	W d    n1 s0    Y  |j|t	j |	gd t j$% r| &t'd8 |jt	j
jdt	j t	j# gd W d    n1 s
0    Y  d S )Nrc  r  rT   r=  Zbinary_accuracyr  r   r   r   r   r  c                 S   s   g | ]
}|j qS r+   )	variables)rR  metricr+   r+   r,   rS    r   zCTestModelCapturesStrategy.test_fit_and_evaluate.<locals>.<listcomp>zAll metrics must be created in)(rv   rw   rx   ry   rk   r\   r   r   rn  r    rf  rg  rh  r   r\  r  r   r   Z
iterationsnumpyrU  r   r   ri  r  rj  r  nestflattenr   
assertTrueextendedZvariable_created_in_scopeZ_weightsrr  ZBinaryAccuracyr   Zhas_strategyr   r   )
r2   r_   r]   r*   r   rk  rl  Zmetric_varsvarrs  r+   r+   r,   rm  x  sd    
,





*
z/TestModelCapturesStrategy.test_fit_and_evaluatec                    s  t j  d} fdd}| }tjdd }}|j||dd || | F | }|	|  
|jj  t|jjd  W d    n1 s0    Y  |  | }W d    n1 s0    Y  |	|  
|jj  t|jjd  d S )	NZckptc                     sD   t jt jdg} | jddd | d dg  | jj	 | S )Nrj   rF  r   )r   r   )
r    r!   r"   r$   r%   r   r   ZassertEmptyr   weightsr)   r   r+   r,   create_model  s    
z>TestModelCapturesStrategy.test_optimizer.<locals>.create_model)rj   rj   r  rj   r  r   )r  r  r  r  rv   rk   r   Zsave_weightsr   Zload_weightsZassertNotEmptyr   rz  rw  r   Zis_distributed_variable)r2   r_   temp_dirr{  r*   rf   rg   r+   r   r,   test_optimizer  s0    



"
$

z(TestModelCapturesStrategy.test_optimizerN)r9   r:   r;   r,  rv   r   r   r   rK  r   r   r   rm  Zmirrored_strategy_with_one_cpur}  r+   r+   r+   r,   rq  u  s   


I


rq  __main__)N)N)Tr,  r  rt  rt   Ztensorflow.compat.v2r   r   rv   Zabsl.testingr   r    r   Zkeras.distributer   r   r   r   Z&keras.distribute.strategy_combinationsr   r	   r
   r   r   Zkeras.enginer   Zkeras.mixed_precisionr   Zkeras.optimizers.optimizer_v2r   r   Zkeras.testing_infrar   Zkeras.utilsr   r   Z-tensorflow.python.distribute.cluster_resolverr   rV   rU   r&   r(   r-   r?   rQ   rX   r`   rh   rn   r{   r|   r   r   r   r   r   r   r   r   r   r   r  r  r   r   ZTestCaser   rL  r  r  Zrun_all_without_tensor_float_32r  rZ  r]  r_  r`  r>   rn  rq  r9   Zenable_v2_dtype_behaviorr   r   Zmulti_process_runnerZ	test_mainr+   r+   r+   r,   <module>   s   	
.
		
B

    i

      F

3:

      A#

C 
