a
    Sic                     @   s   d Z ddlZddlm  mZ ddlmZ ddlm	Z
 ddlmZ ddlmZ ddlmZ e edg d	G d
d deZdS )z,Sharpness Aware Minimization implementation.    N)data_adapter)deserialize)Model)generic_utils)keras_exportz4keras.models.experimental.SharpnessAwareMinimization)v1c                       s`   e Zd ZdZd fdd	Zdd Zdd	 Z fd
dZed fdd	Z	dd Z
dd Z  ZS )SharpnessAwareMinimizationa\  Sharpness aware minimization (SAM) training flow.

    Sharpness-aware minimization (SAM) is a technique that improves the model
    generalization and provides robustness to label noise. Mini-batch splitting
    is proven to improve the SAM's performance, so users can control how mini
    batches are split via setting the `num_batch_splits` argument.

    Args:
      model: `tf.keras.Model` instance. The inner model that does the
        forward-backward pass.
      rho: float, defaults to 0.05. The gradients scaling factor.
      num_batch_splits: int, defaults to None. The number of mini batches to
        split into from each data batch. If None, batches are not split into
        sub-batches.
      name: string, defaults to None. The name of the SAM model.

    Reference:
      [Pierre Foret et al., 2020](https://arxiv.org/abs/2010.01412)
    皙?Nc                    s$   t  j|d || _|| _|| _d S )N)name)super__init__modelrhonum_batch_splits)selfr   r   r   r
   	__class__ e/var/www/html/django/DPS/env/lib/python3.9/site-packages/keras/models/sharpness_aware_minimization.pyr   5   s    z#SharpnessAwareMinimization.__init__c              	   C   s6  t |\}}}| jdur8t|| j}t|| j}n|g}|g}g }g }t||D ]x\}	}
g }t &}| |	}| |
|}W d   n1 s0    Y  |	| | jj
}|||}| |}| j|d  }t||D ].\}}|| }| ||tj  |	| qt $}| |	}| |
|}W d   n1 sF0    Y  |||}t|dkr|D ]}|	|g qnn t||D ]\}}|	| qt||D ] \}}| || tj  qqVg }|D ]}|	tj|dd q| jt|| tj|dd}| j||| dd | jD S )a  The logic of one SAM training step.

        Args:
          data: A nested structure of `Tensor`s. It should be of structure
            (x, y, sample_weight) or (x, y).

        Returns:
          A dict mapping metric names to running average values.
        Ng-q=r   )axisc                 S   s   i | ]}|j | qS r   )r
   result).0mr   r   r   
<dictcomp>       z9SharpnessAwareMinimization.train_step.<locals>.<dictcomp>)r   unpack_x_y_sample_weightr   tfsplitzipGradientTaper   compiled_lossappendtrainable_variablesgradient_gradients_order2_normr   _distributed_apply_epsilon_w
distributeget_strategylen
reduce_sum	optimizerapply_gradientsconcatcompiled_metricsupdate_statemetrics)r   dataxysample_weightZx_splitZy_splitZgradients_all_batchesZpred_all_batchesZx_batchZy_batchZepsilon_w_cachetapepredlossr"   	gradientsZgradients_order2_normscaler#   variable	epsilon_wZgradient_all_batchesr   r   r   
train_step;   s`    



*


,
z%SharpnessAwareMinimization.train_stepc                 C   s
   |  |S )zForward pass of SAM.

        SAM delegates the forward pass call to the wrapped model.

        Args:
          inputs: Tensor. The model inputs.

        Returns:
          A Tensor, the outputs of the wrapped model for given `inputs`.
        )r   )r   inputsr   r   r   call   s    zSharpnessAwareMinimization.callc                    s(   t   }|t| j| jd |S )N)r   r   )r   
get_configupdater   serialize_keras_objectr   r   )r   configr   r   r   r>      s    

z%SharpnessAwareMinimization.get_configc                    s2   t |}t|d|d}||d< t ||S )Nr   )custom_objects)copydeepcopydeserialize_layerpopr   from_config)clsrA   rB   r   r   r   r   rG      s    

z&SharpnessAwareMinimization.from_configc                 C   sT   t tj tjjjtjjjfrFdd }tjjj	|tj || n
|
| d S )Nc                 S   s   | j j|dd |fdd d S )Nc                 S   s
   |  |S N)
assign_add)r1   r2   r   r   r   <lambda>   r   zcSharpnessAwareMinimization._distributed_apply_epsilon_w.<locals>.distribute_apply.<locals>.<lambda>F)argsgroup)extendedr?   )strategyvarr:   r   r   r   distribute_apply   s    zQSharpnessAwareMinimization._distributed_apply_epsilon_w.<locals>.distribute_apply)
isinstancer   r&   r'   experimentalParameterServerStrategyCentralStorageStrategy__internal__interimmaybe_merge_callrJ   )r   rP   r:   rO   rQ   r   r   r   r%      s    
z7SharpnessAwareMinimization._distributed_apply_epsilon_wc                 C   s   t t dd |D }|S )Nc                 S   s   g | ]}|d urt |qS rI   )r   norm)r   gradr   r   r   
<listcomp>   r   zESharpnessAwareMinimization._gradients_order2_norm.<locals>.<listcomp>)r   rY   stack)r   r7   rY   r   r   r   r$      s    z1SharpnessAwareMinimization._gradients_order2_norm)r	   NN)N)__name__
__module____qualname____doc__r   r;   r=   r>   classmethodrG   r%   r$   __classcell__r   r   r   r   r      s   F
	r   )r`   rC   Ztensorflow.compat.v2compatv2r   keras.enginer   keras.layersr   rE   Zkeras.modelsr   keras.utilsr    tensorflow.python.util.tf_exportr   register_keras_serializabler   r   r   r   r   <module>   s   
