a
    Sic&                     @   sd   d Z ddlm  mZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ edG dd	 d	ejZdS )
zNadam optimizer implementation.    N)backend_config)optimizer_v2)learning_rate_schedule)keras_exportzkeras.optimizers.Nadamc                       sb   e Zd ZdZdZd fdd	Zd	d
 Zdd Z fddZdddZ	dddZ
 fddZ  ZS )Nadama  Optimizer that implements the NAdam algorithm.
    Much like Adam is essentially RMSprop with momentum, Nadam is Adam with
    Nesterov momentum.

    Args:
      learning_rate: A Tensor or a floating point value.  The learning rate.
      beta_1: A float value or a constant float tensor. The exponential decay
        rate for the 1st moment estimates.
      beta_2: A float value or a constant float tensor. The exponential decay
        rate for the exponentially weighted infinity norm.
      epsilon: A small constant for numerical stability.
      name: Optional name for the operations created when applying gradients.
        Defaults to `"Nadam"`.
      **kwargs: keyword arguments. Allowed arguments are `clipvalue`,
        `clipnorm`, `global_clipnorm`.
        If `clipvalue` (float) is set, the gradient of each weight
        is clipped to be no higher than this value.
        If `clipnorm` (float) is set, the gradient of each weight
        is individually clipped so that its norm is no higher than this value.
        If `global_clipnorm` (float) is set the gradient of all weights is
        clipped so that their global norm is no higher than this value.

    Usage Example:
      >>> opt = tf.keras.optimizers.Nadam(learning_rate=0.2)
      >>> var1 = tf.Variable(10.0)
      >>> loss = lambda: (var1 ** 2) / 2.0
      >>> step_count = opt.minimize(loss, [var1]).numpy()
      >>> "{:.1f}".format(var1.numpy())
      9.8

    Reference:
      - [Dozat, 2015](http://cs229.stanford.edu/proj2015/054_report.pdf).
    TMbP??+?Hz>c                    s   | dd|d< |d|}t|tjr0tdt j|fi | | d|d| | d| j	 | d| | d| |pt
 | _d | _d S )	Nschedule_decaygMbp?decaylrzdThe Nadam optimizer does not support tf.keras.optimizers.LearningRateSchedules as the learning rate.learning_ratebeta_1beta_2)popget
isinstancer   LearningRateSchedule
ValueErrorsuper__init__
_set_hyper_initial_decayr   epsilon_m_cache)selfr   r   r   r   namekwargs	__class__ _/var/www/html/django/DPS/env/lib/python3.9/site-packages/keras/optimizers/optimizer_v2/nadam.pyr   A   s    
zNadam.__init__c                 C   sp   |d j j}| jd u r@| jdg |ddtjjd| _| j| j |D ]}| 	|d qD|D ]}| 	|d qZd S )Nr   momentum_cacheonesF)shapedtypeinitializer	trainableaggregationmv)
r&   
base_dtyper   
add_weighttfVariableAggregationONLY_FIRST_REPLICA_weightsappendadd_slot)r   var_list	var_dtypevarr!   r!   r"   _create_slots^   s    
zNadam._create_slotsc                 C   s@  t | d|}t | d|}t | d|}t | jd |}t | jd |}t d|}	|ddt |	| j|    }
|ddt |	| j|    }t | j||
 }|| jj	u rt t j
jj| j|| jd	}|| }t|| t | j||||
|d| d| d|
 d| d| dt || d
|||f< d S )Nr   r   r         gQ?g      ?g      ?use_locking)lr_tneg_lr_tr   beta_1_tbeta_2_tm_tm_t_1one_minus_beta_1_tone_minus_beta_2_tone_minus_m_tone_minus_m_schedule_newone_minus_m_schedule_nextv_t_prime_denominator)r.   identity
_get_hypercast
iterationspowr   _m_cache_readr   r&   compatv1assign_use_lockingdictconvert_to_tensorr   )r   
var_devicer5   apply_stater<   r>   r?   
local_step	next_step
decay_baser@   rA   m_schedule_newm_schedule_nextr!   r!   r"   _prepare_localr   sF    
zNadam._prepare_localc                    s   t | j| _t |S )N)r.   rH   r   rM   r   _prepare)r   r4   r   r!   r"   r\      s    zNadam._prepareNc                 C   s  |j |jj }}|pi ||fp,| ||}| |d}| |d}||d  }	|d | |d |  }
tjjj	||
| j
d}
|
|d  }|d | |d	 t|  }tjjj	||| j
d}||d
  }|d |	 |d |  }||d | t||d    }tjjj	||| j
djS )Nr*   r+   rE   r>   rB   r:   rF   r?   rC   rG   rD   rA   r<   r   )devicer&   r,   r   _fallback_apply_stateget_slotr.   rN   rO   rP   rQ   squaresqrtop)r   gradr6   rU   rT   r5   coefficientsr*   r+   g_primer@   	m_t_primev_t	v_t_primem_t_barvar_tr!   r!   r"   _resource_apply_dense   s<    




zNadam._resource_apply_densec                 C   s  |j |jj }}|pi ||fp,| ||}| |d}| |d}	||d  }
||d  }tjjj	|||d  | j
d}t|g* | |||}t||}W d    n1 s0    Y  ||d  }|d |
 |d	 |  }|| |d
  }tjjj	|	|	|d  | j
d}t|g* | |	||}t||}W d    n1 sR0    Y  ||d  }t||d  }| |||d | | }tj|||g S )Nr*   r+   rE   rB   r>   r:   rF   rD   rA   rC   r?   rG   r   r=   )r]   r&   r,   r   r^   r_   r.   rN   rO   rP   rQ   control_dependencies_resource_scatter_addgatherra   group)r   rc   r6   indicesrU   rT   r5   rd   r*   r+   re   m_scaled_g_valuesr@   	m_t_slicerf   ri   v_scaled_g_valuesrg   	v_t_slicerh   v_prime_sqrt_plus_eps
var_updater!   r!   r"   _resource_apply_sparse   sH    
*

,zNadam._resource_apply_sparsec                    s:   t   }|| d| j| d| d| jd |S )Nr   r   r   )r   r   r   r   r   )r   
get_configupdate_serialize_hyperparameterr   r   )r   configr   r!   r"   rx      s    
zNadam.get_config)r   r   r	   r
   r   )N)N)__name__
__module____qualname____doc___HAS_AGGREGATE_GRADr   r7   r[   r\   rk   rw   rx   __classcell__r!   r!   r   r"   r      s   "     )

/r   )r   tensorflow.compat.v2rN   v2r.   kerasr   keras.optimizers.optimizer_v2r   keras.optimizers.schedulesr    tensorflow.python.util.tf_exportr   OptimizerV2r   r!   r!   r!   r"   <module>   s   