a
    ==icV                     @   s   d dl mZmZ d dlmZ d dlmZ d dlmZm	Z	m
Z
mZmZmZmZmZmZ d dlZd dlZd dlZd dlZd dlZd dlZeeZdd Zddd	Zd
d Zdd Zdd ZG dd deZ dS )    )FP16_ENGINES	Optimizer)get_current_scope)schema)	get_categorical_limitget_keyIdListIdScoreListIdListWithEvictedIdScoreListWithEvictedLayerPsParam
ModelLayeralmost_equal_schemasNc                 C   sf   t | tr^t| dr^td| j | jtv rFtd| j dS td| j dS ndS d S )Nenginez/Attempting to set trainer version for engine {}z"Setting FP16 trainer for engine {}fp16z"Setting FP32 trainer for engine {}fp32)
isinstancer   hasattrloggerinfoformatr   r   )Z	optim_def r   s/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/caffe2/python/layers/sparse_lookup.py"get_trainer_version_based_on_optim    s    

r   c                 C   s   | dv sJ d | | dkr|d urz|d urz|d urz||k rVtd ||| d} |d dkrtd || d} ntd	 || |||| S )
N>   r   fused_uint8rowwiseuint8rowwisefused_uint4rowwiser   -Unexpected version of sparse_lookup layer {0}r   zK{} fall back to uint8 because lookup table size {} < min_blob_size_4bits {}r         zM{} fall back to uint8 because lookup table dimension {} is not divisible by 2zWhen 4 bit quantization is enabled for {}, (i.e., Sparse lookup predictor version:{}), requires arguments blob_size:{}, min_blob_size_4bits:{}, embedding_dim:{})r   r   r   
ValueError)version	blob_sizemin_blob_size_4bitsembedding_dimsparse_feature_namer   r   r   #get_sparse_lookup_predictor_version/   sJ    
r&   c                 C   s   | dv sJ d | | S )N>   r   r   r   )r   r!   r   r   r   !get_sparse_lookup_trainer_versionb   s    
r(   c                 C   s
   t | tS N)r   r   input_recordr   r   r   _is_id_listg   s    r,   c                 C   s   t | tddS )NFZcheck_field_types)r   r	   r*   r   r   r   _is_id_score_listk   s    r.   c                       s   e Zd Zg dZg dZdgZg dZd& fd	d
	Zdd Zdd Z	dd Z
d'ddZdd Zdd Zd(ddZdd Zdd Zd)d d!Zd"d# Zd$d% Z  ZS )*SparseLookup)	Z
LogMeanExpZ	LogSumExpZMaxMeanSumWeightedSumWeightedMeanSqrtNone)PositionWeightedRecencyWeightedr0   r1   r2   r3   r5   Float16UniformFill)r1   r0   r4   r6   r7   Nsparse_lookupF      ?c              	      s  t t| j|||fi | t| j | _td| j  t|t	rJ|g}t|t
stt|tstJ dt|| j|dkrt| jsJ dt| j| j| j | _nt|dkrt| jsJ d| j| j | _nD|	r t| jsJ d| j|dv sJ d	|| j | _|| _|	| _t| j}|d
ksRJ d| j||| _|g| | _t|| _|
| _|  }|p|| _d | _t| jtstj| jt ddr| jj!| _| jdkrL| j| j"v s|	sJ d| j| j"| jd
 dkrd| jd f| _| jd
 | j#v s:J d| jd
 | j#|d u sLJ d| jj$j%rf| jj$j%j&}nd }| j'd| j| j|t(| j|d|d| _)| jr| j'd|| j|j*d d| _+dddif| _,| j'dg | j,|j*d| _-t.t/j0|f| 1d| _2d S ) Nz"Setup the sparse lookup layer for zHUnexpected type for inner_shape, expected list or tuple, got {0} for {1}r6   z~PositionWeighted only support IdScoreList, but got {} for {}please use PositionWeighted layer to convert IdList to IdScoreListr7   zNRecencyWeighted only supports IdScoreList, while the sparse feature {} is not.zSUse_external_weights only supports IdScoreList, while the sparse feature {} is not.)r1   r2   zHUse_external_weights only supports Sum reducer, while the reducer is {}.r   z0{} should have categorical limit > 0, but got {}Fr-   r   a  Fp16 training is enabled. The reducer specified is not supported. Got {}. Supported reducers: {}. Right now, in general, sum, mean, positional pooling are supported. Attention is not. Please check if there is fp16 trained sparse features using advanced pooling.UniformFillr8   r   zjFp16 training is enabled. Init op for weight parameter must be fp16 compatibale. Got {}. Supported ops: {}z'Regularizer is not compatible with fp16w)
sparse_keyZaverage_length)
param_nameshapeinitializer	optimizerZps_paramregularizer
reinit_vec)r>   r?   r@   rA   rB   ZConstantFillvalueg        
scale_bias)r>   r?   r@   rA   output)3superr/   __init__r   r+   r=   r   r   r   intlisttupler   typer.   reprvaluesexternal_weightsreduceruse_external_weightsr   	input_dimr?   r   trainer_version#uniform_weight_init_scale_numerator_get_default_init_opweight_initevicted_valuesr   Zequal_schemasr
   r   Z_evicted_values_fp16_compatible_reducers_fp16_compatible_init_op_typeslengthsmetadataZexpected_valueZcreate_paramr   r<   ZNoOptimrC   Zscale_bias_initrE   ZScalarnpfloat32Zget_next_blob_referenceoutput_schema)selfmodelr+   Zinner_shaperP   rV   Zweight_optimnamerB   rQ   rT   kwargsrR   Zdefault_init_opZ
avg_length	__class__r   r   rH      s    



	
zSparseLookup.__init__c                 C   s   t tj| jd S )N   )	functoolsreduceoperatormulr?   r_   r   r   r   get_memory_usage  s    zSparseLookup.get_memory_usagec                 C   s   | j gS r)   )r<   rj   r   r   r   get_fp16_compatible_parameters
  s    z+SparseLookup.get_fp16_compatible_parametersc                 C   s$   t | jdks| jd dk r dS dS )Nr   r      FT)lenr?   rj   r   r   r   support_8bit  s    zSparseLookup.support_8bitTc                 C   sH   |   sg S |r(tdd}|| jgS tdd}|| j| jgS d S )NRowwiseQuantized8BitsWeightr<   zw, scale_bias)ro   collections
namedtupler<   rE   )r_   Zfusedrp   r   r   r   get_8bits_compatible_parameters  s    z,SparseLookup.get_8bits_compatible_parametersc                 C   s`   t | j| j }| jdkr.d| |df}n.| jdkrJd| |df}ntdt| j|S )Nr   r;   )minmaxr   r8   zATrain version {} is not currently supported for sparse feature {})mathsqrtrT   rR   rS   NotImplementedErrorr   r=   )r_   scaleZdefault_weight_initr   r   r   rU   "  s    

z!SparseLookup._get_default_init_opc                 C   s   |dkr| | j|g|S |dkr@| | j|gd}|||S |dkr|| | j|gd}| | j|gd}|||g|S |dkr| | j|gd}|||S |dkr| | j|gd}|||S dd	|| j d S )
Nr   r   
gathered_wr   gathered_scale_biasr   r   z1Unsupported version of operators in SparseLookup !layer: {0} for sparse feature {1})	ZGatherr<   ZHalfToFloatrE   ZRowwise8BitQuantizedToFloatZ Fused8BitRowwiseQuantizedToFloatZ Fused4BitRowwiseQuantizedToFloatr   r=   )r_   netr!   
in_indicesoutrz   r{   r   r   r   _gather_wrapper2  s2    zSparseLookup._gather_wrapperr   c           	      C   s   | j ||| j g}d| }|dv rj|dkrN|dkrN|j|| j |dd q|||| j |d n|dkr|t|| j	 ||d	 || j  nZ|d
kr||d || j  n6|dkr||d || j  ndd
|| j d S )NSparseLengthsr   r   r2   r   ZFP16)grad_on_weightsr   r   r   8BitsRowwiser   Fused8BitRowwiser   Fused4BitRowwise0Unsupported version of operator in SparseLookUp r|   )r<   r+   rZ   ZSparseLengthsWeightedSumr^   field_blobs__getattr__insertrn   rE   r   r=   )	r_   r~   weightsrP   r}   r!   r   op_input
layer_namer   r   r    _sparse_lengths_weighted_reducerQ  sJ    



z-SparseLookup._sparse_lengths_weighted_reducerc                 C   s  | j | jv s J d| j | j| j dv r$| j| j | j g}| j dkrVd| _ n| j dkrfd| _ d| j  }|dv r|||| j	
  n|d	kr|t|| j ||d
 || j	
  n\|dkr||d || j	
  n8|dkr||d || j	
  ndd|| j n| j dkrj|j| j g|dgdd}| | j |d|| n| j dkr| ||| j | j	
  nX| ||| j d}|| j || j d }|d| j  ||g| j	
  d S )Nz&Unsupported reducer: {} for ID_LIST {})r1   r0   r2   r3   r2   r1   r3   r0   r   r   r   r   r   r   r   r   r   r|   r4   Zlengths_sqrtg      ?)powerr5   
table_rowsZ_sidZSortedSegmentRange)rP   _id_list_supported_reducersr   r=   r<   r+   itemsrZ   r   r^   r   r   rn   rE   ZLengthsToWeightsZNextScopedBlobr   r   ZLengthsToSegmentIds)r_   r}   r!   r   r   Zsqrt_weightr   Zsegment_idsr   r   r   _add_ops_id_list  s    








zSparseLookup._add_ops_id_listc                 C   s  | j | jv s J d| j | j| j dv rN| | j | j | j || nB| j dv s^| jr| j| j | j	d||dd n| j dv rR| j
| j | j g}d| j  }|d	v r|||| j  n|d
kr||d || j  n^|dkr||d || j  n8|dkr>||d || j  ndd|| j n>| j dkr|| ||| j | j  ndd| j | j d S )Nz,Unsupported reducer: {} for ID_SCORE_LIST {})r2   r3   )r6   r7   r2   r   r   )r1   r0   r   r   r   r   r   r   r   r   r   r|   r5   z9Only Sum, Mean, None are supported for IdScoreList input.z.Trying to create with {} for sparse feature {})rP   !_id_score_list_supported_reducersr   r=   r   r+   keysrN   rQ   rO   r<   rZ   r   r^   r   r   )r_   r}   r!   r   r   r   r   r   _add_ops_id_score_list  sl    







z#SparseLookup._add_ops_id_score_listr   c                 C   sn   | j r*|r*|| j| j  | jg| jg t| jrD| j||d n&t| jr^| j	||d nd
| jd S )Nr'   zUnsupported input type {0})rW   ZCopyRowsToTensorr<   getrC   r,   r+   r   r.   r   r   )r_   r}   r!   is_trainr   r   r   _add_ops  s    


zSparseLookup._add_opsc                 C   s   | j || jdd d S )NTr   )r   rS   )r_   r}   r   r   r   add_train_ops  s    zSparseLookup.add_train_opsc                 C   s~   t  tjddi}| jd | jd  }t|d |d|v rB|d nd | jd | jd}|  sj|dv rjd}| j||d	d
 d S )Nr!   r   r   r   r#   )r"   r#   r$   r%   >   r   r   r   r   Fr   )r   r   r&   __name__r?   r=   ro   r   )r_   r}   version_infoZlookup_table_blob_sizer!   r   r   r   add_ops  s     

zSparseLookup.add_ops)NNr9   NFr:   )T)r   )r   T)r   
__module____qualname__r   r   rY   rX   rH   rk   rl   ro   rs   rU   r   r   r   r   r   r   r   __classcell__r   r   rc   r   r/   q   s.       
& 
5I8
r/   )NNNN)!Zcaffe2.python.optimizerr   r   Zcaffe2.python.helpers.arg_scoper   Zcaffe2.pythonr   Zcaffe2.python.layers.layersr   r   r   r	   r
   r   r   r   r   rq   rf   loggingrv   numpyr\   rh   	getLoggerr   r   r   r&   r(   r,   r.   r/   r   r   r   r   <module>   s(   ,
    
3