a
    8Sicg                    @   sl  d Z ddlZddlZddlZddlZddlmZmZmZm	Z	 ddl
Z
ddlm  mZ ddlZ
ddlZ
ddl
mZ ddlmZ ddlmZ ddlmZ dd	 Zd
d Zdd Zdd Zdd ZdfddZdgddZdhddZdd Zdd Zedddddidd Z eddd!d"d# Z!d$d% Z"d&d' Z#d(d) Z$d*d+ Z%d,d- Z&d.d/ Z'edd0d1d2 Z(edd0d3d4 Z)d5d6 Z*d7d8 Z+d9d: Z,d;d< Z-edddd=d=d>d? Z.d@dA Z/dBdC Z0dDdE Z1dFdG Z2dHdI Z3dJdK Z4dLdM Z5dNdO Z6dPdQ Z7dRdS Z8ej9dTdUddVdWdX Z:dYdZ Z;d[d\ Z<d]d^ Z=djd_d`Z>dadb Z?dkdcddZ@e@dedfZAe@dgdhZBe@didjdkdlZCedd0dmdndo ZDdpdq ZEdrds ZFdtd= ZGdudv ZHdwdx ZIeddd0dyddzd{ ZJedddd0d0d0dd0d0	d|d} ZKdld~dZLedd0d0dd ZMedddd ZNdd ZOdd ZPedd0d0d0dmddZQedddd0dnddZRdoddZSeddd0d0dpddZTdqddZUedd0d0drddZVedd0ddd ZWdd ZXdsddZYdd ZZdd Z[dd Z\dd Z]e9dTdd Z^e9dTdd Z_dd Z`dd Zadd Zbedd=d=dd ZcdtddZdedd0dd Zeedd0dmduddZfdd Zgdd Zhdd Zieide
jjjkjljmddkdZneide
jjjkjljoddkdZpeide
jjjkjljqddkdZreide
jjjkjljmddTdZseide
jjjkjljoddTdZteide
jjjkjljqddTdZuddƄ Zvevde
jjjkjljmZwevde
jjjkjljoZxevde
jjjkjljqZydvdd˄Zzezdde
jjjkjljmZ{ezdde
jjjkjljoZ|ezdde
jjjkjljqZ}ezdde
jjjkjljmesZ~ezdde
jjjkjljoetZezdde
jjjkjljqeuZddՄ Zddׄ Zddل Zddۄ Zdd݄ Zdd߄ ZeZeZeZeZeZeZdd Zdd ZedddZedddZedddZedddZedddZedddZdd Zdd Zdd Zdd Zdd Zdd Zdd Zedd Zdd Zd d Zdd Zdd Zedd Zedd	 Zd
d Zdd Zdd Zeddd Zeddd Zeddd Zdd Zdd Zedddd0dwddZedd0dmdxddZeddddddd0dd0d0d0d0d0dydd Zeddddddd0d!d" Zeddddddd0d#d$ Zeddddddd0d%d& Zeddddddd0dd'd( Zeddddddd0dd)d* Zeddddddd0dd+d, Zedddddd0ddd0	d-d. Zedddddd0d/d0 Zedddddd0ddd0	d1d2 Zedd0d0d0d3d4 Zedd=d=d=d5d6 Zd7d8 Zedd0dd9d: Zd;d< Zd=d> Zd?d@ ZedddydydzdAdBZdCdD Zeddd0ddEdF ZdGdH ZdIdJ ZdKdL ZÐdMdN ZĐdOdP ZŐdQdR ZƐdSdT ZǐdUdV ZedddWdX ZedddYdZ Zʐd{d[d\Zːd]d^ Z̐d|d_d`Z͐dadb Zeddd0dcdd Zeddd0dedf Zeddd0dgdh Zѐdidj Zeddd0dkdl ZӐdmdn ZeԐdoZeԐdpZeԐdqZeZeZeZeZedd=dd0drds Zedddd0dtdu Zedd0d0dvdw Zedd0d0d0dxdy Zej D ]6\ZZdzeZedd0eejee e< qedd0ddddd}d{d|Zedd0ddddd~d}d~ZdddZdd ZdddZdddZedd0ddddddZedd0dddddddZdddZedd0ddddddZedd0dddddddZdddZdddZdddZdddZdd Zdd Zeddddd Zeddd Zej9dTdUddVeddd Zeddd Zedddd Zedddd Zdd Z edd0dd Zedd0d0dmdddZdd Zedd0d0d0d0dmdddZdd Zdd ZdddZedd0dd Zedd0dd Z	dddZ
edddd0d0dd0d0d0	dd Zeddddd0d0dd0d0	dd ZdÐdĄ ZdŐdƄ ZdǐdȄ ZedɃZedʃZed˃Zedd0d̐d̈́ Zdΐdτ Zedd0dАdф Zeddd0dҐdӄ Zeddd0d=ddԐdՄ Zd֐dׄ Zdؐdل ZddڐdۄZddܐd݄Zedddd0dmdސd߄ ZdddZeddd Zeddd Ze9dTdkdkedd0d0dd Z eddd Z!dddZ"eddd Z#dd Z$dd Z%edd0d0d0dd Z&dd Z'dd Z(edd0dddd Z)edd0dddd Z*dd Z+dd Z,d d Z-edd Z.dd Z/edd0dddddZ0eddd0d0dd	 Z1d
d Z2dd Z3dd Z4dd Z5eddd0dd Z6dd Z7dd Z8dd Z9dd Z:dd Z;edddd0ddd Z<edddd0dd d! Z=edddd0dd"d# Z>eddd0dd%d&Z?eddd0dd'd(Z@edd0dyddd)d*ZAd+d, ZBedd!deeC d-d.d/ZDd0d1 ZEedd!de
jjFeCd2d3d4ZGedd0dddd0d5d6 ZHeddd0d7d8 ZId9d: ZJd;d< ZKd=d> ZLd?d@ ZMdAdB ZNdCdD ZOeddd0dydEdF ZPedddd0ddGdHZQdIdJ ZRdKdL ZSdMd ZTeddyd0ddddddNdOZUdPdQ ZVdRdS ZWedddTdU ZXddVdWZYeddddXdY ZZeddd0ddZd[Z[dd^d_Z\d`da Z]G dbdc dcZ^G ddde deZ_dS (  zhThis file exports ONNX ops for opset 9.

Opset 9 is supported by ONNX release 1.4.1
release on 01/23/19
    N)ListOptionalTupleUnion)_C)_patch_torch)symbolic_helper)GLOBALSc                 C   s   |  d}|tj  |S )Nprim::Constant)opsetTyper   OptionalTypeofTensor)gn r   V/var/www/html/django/DPS/env/lib/python3.9/site-packages/torch/onnx/symbolic_opset9.pyunused9   s    
r   c                 C   s   |  d|S NShaper   r   inputr   r   r   _shape_as_tensor?   s    r   c                 C   s.   t |tr"| jdg|R ddi}t| ||S )NConcataxis_ir   )
isinstancelistr   reshape)r   r   shaper   r   r   _reshape_from_tensorC   s    
r    c                 C   s   t | ||S N)r   _reshape_helperr   selfr   r   r   r   r   I   s    r   c                 C   s   |  d|}t| ||S r   r   r   r   r$   otherr   r   r   r   
reshape_asM   s    r(   c                 C   sV   t |r$t |r$t ddddS |rHt t |dkrHt ddS | d||S )NAdd	      z)Add between list of tensors not supported   add
alpha != 1)r   	_is_value_is_tensor_list _onnx_opset_unsupported_detailed_scalar_maybe_get_scalar_unimplementedr   r   r$   r'   alphar   r   r   r-   R   s    r-   c                 C   s2   |r$t t |dkr$t ddS | d||S )Nr,   subr.   Sub)r   r2   r3   r4   r   r5   r   r   r   r7   ^   s    r7   c                 C   s   t | |||dS )N)r6   )r7   r5   r   r   r   rsube   s    r9   c                 C   s   |  d||S NMulr   r   r$   r'   r   r   r   muli   s    r=   c                 G   s0   t |dkrt| ||S t| ||g|R  S d S Nr   )lentrue_divide_div_rounding_mode)r   r$   r'   argsr   r   r   divm   s    rC   vf      ?c              	   C   s2   | j dt|gd}t| |t| t| |||S )NConstantvalue_t)r   torchtensorr-   r=   )r   r$   tensor1tensor2valueZ
value_tensr   r   r   addcmult   s    rO   sc                 C   sP   |d u rt | ||S |dkr(t| ||S |dkr<t| ||S td| dd S )NfloortrunczUnsupported rounding mode: "z$". Expected None, "floor" or "trunc")r@   _floor_divide_trunc_divideRuntimeError)r   r$   r'   rounding_moder   r   r   rA   z   s    
rA   c                 C   s   |  d||}| j d|tjd d}|  }|d urt|st|  d urtt|rt| j d|tjd d}q| j d|tj| d}n| j d|tjd d}|S )NDivCastLongto_iFloat)r   r   cast_pytorch_to_onnxtype
scalarType_is_fp)r   r$   r'   outscalar_typer   r   r   rT      s     rT   c                 C   s   t |st |r,t| ||}| d|S | d||}| jdtjdtjdd}| dt | ||t | ||}| d|| d	||}| d
|| d| d||}| jdtjdtjdd}	| d	||	}
| d||
S d S )NFloorrW   rG   r   dtyperH   Xorr8   r;   AndNotEqualr,   )r   r`   r@   r   rJ   rK   int64
_lt_helper)r   r$   r'   ra   rC   zeronegativemodZ
fixup_maskonefixupr   r   r   rS      s     rS   c                 C   s   t | ||S r!   )rT   r<   r   r   r   floor_divide   s    rq   c                 C   s   t | ||S r!   )rq   r<   r   r   r   floordiv   s    rr   c                 C   s   t |st |r"| d||S t }t jd }|tju sL|tju sLJ t tju rdt jd }| jd||d}| jd||d}| d||S )a  Division where both inputs are cast to floating types

    If both inputs are floating, performs div as usual
    If only one input is a floating type, the other input is cast to its type
    If neither input is a floating type, both inputs are cast to the default scalar type
    rW   r\   DoublerX   rZ   )r   r`   r   rJ   get_default_dtyper]   floatdouble)r   r$   r'   rb   Zonnx_scalar_typer   r   r   r@      s    

r@   c                 C   s,   t |s | jd|t jd d}| d|S )NrX   r\   rZ   
Reciprocal)r   r`   r   r]   r   r$   r   r   r   
reciprocal   s    
ry   ic                 C   s"   t |}| jdg|R d|iS )Nr   r   r   _unpack_listr   )r   tensor_listdimtensorsr   r   r   cat   s    
r   c                    s2    fddt |D }jdg|R d iS )Nc                    s   g | ]}t | gqS r   r   _unsqueeze_helper.0tr~   r   r   r   
<listcomp>   s   zstack.<locals>.<listcomp>r   r   r{   )r   r}   r~   Z
unsqueezedr   r   r   stack   s    r   c                 C   s   |S r!   r   rx   r   r   r   _list   s    r   c                 C   s,   | j dtdgd}| j d|||dddS )NrG   r,   rH   Gemm        rF   beta_falpha_f)r   rJ   rK   )r   r$   r'   Cr   r   r   mm   s    r   c                 C   s   |  d||S NMatMulr   r<   r   r   r   bmm  s    r   c                 C   s   |  d||S r   r   r<   r   r   r   matmul	  s    r   r   c              	   C   s\  d }t |}t |}t |}	|d ur0|}n|d ur>|}n|	d urJ|	}t |}
t |}dd }|d ur:||
ds||dr:t jt j| }t j| }| d||}|}t |}t |}|dkr| jdt	j
||dd}| d	||}|dkr,| jdt	j
t ||dd}| d	||}| d
||S | jd|||t |t |dS )Nc                 S   s   | d uo| |kS r!   r   )rD   ur   r   r   isNotNoneAnd  s    zaddmm.<locals>.isNotNoneAnd   r   r,   rG   rd   rH   r;   r)   r   r   )r   _try_get_scalar_type_get_tensor_rankscalar_type_to_onnxindexr]   scalar_type_to_pytorch_typer   r2   rJ   rK   )r   r$   mat1mat2betar6   re   
self_dtypeZ
mat1_dtypeZ
mat2_dtypeZ	mat1_rankZ	mat2_rankr   res1res2r   r   r   addmm  sP    




 



r   c                 C   s   |  d|S )NNegr   rx   r   r   r   negB  s    r   c                 C   s   |  d|S NSqrtr   rx   r   r   r   sqrtF  s    r   c              	   C   s$   |  dt| td|t| |S )NrW   r,   )r   r   _if_scalar_type_asrJ   onesr   rx   r   r   r   rsqrtJ  s    r   c                 C   s   |  d|S )NTanhr   rx   r   r   r   tanhP  s    r   c                 C   s   |  d|S )NSinr   rx   r   r   r   sinT  s    r   c                 C   s   |  d|S )NCosr   rx   r   r   r   cosX  s    r   c                 C   s   |  d|S )NTanr   rx   r   r   r   tan\  s    r   c                 C   s   |  d|S )NAsinr   rx   r   r   r   asin`  s    r   c                 C   s   |  d|S )NAcosr   rx   r   r   r   acosd  s    r   c                 C   s   |  d|S )NAtanr   rx   r   r   r   atanh  s    r   Tg      p?)scale
zero_pointc                 C   s   |  d|S )NSigmoidr   rx   r   r   r   sigmoidm  s    r   c                 C   s   |  d|S )NSignr   rx   r   r   r   signr  s    r   c                 C   sP   t |t |ksJ t |dkr<|d dkr<|d dkr<|S | jd||||dS )Nr,   r       Slice)axes_iZstarts_iZends_i)r?   r   )r   r   axesstartsendsr   r   r   _slicev  s    $r   c                 C   s6   |   }|d ur2t|s2|dks2t| |d}|S )NrY   F)r^   r_   r   r`   
_cast_Longr   r$   re   r   r   r   _maybe_cast_reduce_op_input}  s
    r   c                    s   d fdd	}|S )Nc                    sn   t | |}|d u r t| |S  r(dnd}t||dt|dd }} rR|n|g}| j|||dS d S )Nisrz   r~   keepdimr   
keepdims_i)r   r   _handle_reduce_dim_none
_get_constr   )r   r$   r~   r   descdim_listallow_multi_dim_supportonnx_op_namer   r   symbolic  s    
z%_reduce_op_symbolic.<locals>.symbolic)NNr   )r   r   r   r   r   r   _reduce_op_symbolic  s    r   c                    s   t   fdd}|S )Nc                    s\    | g|R  }d }|D ].}|j }t|t|kr|| g|R    S qtd jd S )NzUnknown aten::{} signature)_arg_descriptorsr?   NotImplementedErrorformat__name__)r   rB   	overloadslast_exceptionoverloadarg_descriptorsfnr   r   wrapper  s    z&overload_by_arg_count.<locals>.wrapper)	functoolswraps)r   r   r   r   r   overload_by_arg_count  s    	r   c                    s$   t |  dt fdd}|S )Nr   c                    sL   t ddfdd} r"dnd}t d|ddfdd}||fS )	NrD   nonec                    s\   |   dkr6t|dd}| jd|tj| d}n|   dkrRt dS | |S Nonnx::Constantrz   re   rX   rZ   r
   nodekindr   r   r   r   r4   r   namer   r   r   reduce_nodim  s    z8_reduce_with_dtype.<locals>.reduce.<locals>.reduce_nodimr   rz   c                    s`   |   dkr6t|dd}| jd|tj| d}n|   dkrRt dS | |||S r   r   )r   r$   r~   r   re   r   r   r   
reduce_dim  s    z6_reduce_with_dtype.<locals>.reduce.<locals>.reduce_dimr   
parse_args)r   rB   kwargsr   Zdim_descr   r   r   r   r   r   reduce  s    


z"_reduce_with_dtype.<locals>.reduce)r   r   )onnx_opr   r   r   r   r   r   _reduce_with_dtype  s    r   	ReduceSumsum
ReduceMeanmean
ReduceProdprodFr   r   c                 C   sF   t  r4|  dkr$t tdS | jd||dS t ddd d S )Nr
   re   cumsumdim_ir*   r+   )r   is_caffe2_aten_fallbackr   r   r4   r   at_onnx_opset_unsupported)r   r   r~   re   r   r   r   r     s
    r   c                 C   s8   t  r*t |st ddS | d|S t dS d S )N_sample_dirichlet#We are not able to export generatorr   r   _is_noner4   r  _onnx_unsupportedr   r$   	generatorr   r   r   r    s    
r  c                 C   s8   t  r*t |st ddS | d|S t dS d S )N_standard_gammar  r  r  r   r   r   r
    s    
r
  c                 C   s   | j d|ddS )N	Transpose)r,   r   perm_ir   rx   r   r   r   r     s    c              	   C   s   t |d}t |s,| jdt|d}n2t |r^t | t| |d| jdt	dgd}t j
j}t| ||}t| || jdt	dd}t| | d||||}| d||S )Nr   rG   rH   r   ri   Expand)r   _maybe_get_constr/   r   rJ   
LongTensor_is_packed_listr"   r   rK   
ScalarTypeINT64	ones_liker=   where)r   r$   sizeimplicitre   r   Zneg_onesr   r   r   expand  s    

 r  c                 C   s   t |d}t|tjr|j}|tj}g }t|	 D ]F}t
|||||r:|| | jd|||d}q:| d|}| d||S )Nr   rG   rH   r   r  )r   r  r   rJ   Tensorre   torv   ranger~   equalr   	unsqueeze	expand_asappendr   )r   r$   r'   Zself_t	orig_typedimsdr   r   r   r   r    s    
r  bc                 C   s8   |rt jrtd|dkr*t jr*td | d||S )NzUnsupported: ONNX export of embedding with scale_grad_by_freq=True for training mode. ONNX does not support scaling the gradients.r   zWarning: ONNX export of embedding with padding_idx >= 0 for training mode. ONNX does not support not updating the embedding vector at padding_idx during training.Gather)r	   training_moderU   warningswarnr   )r   weightindicespadding_idxscale_grad_by_freqsparser   r   r   	embedding  s    
r.  c
           
      C   sH   t |st dS t  r:| jd|||d|||||	d
S t dS d S )Nz&embedding_bag  with per_sample_weightsembedding_bag   )outputsZscale_grad_by_freq_iZmode_iZsparse_iZinclude_last_offset_iZpadding_idx_i)r   r  r  r   r  )
r   embedding_matrixr*  offsetsr,  moder-  per_sample_weightsinclude_last_offsetr+  r   r   r   r/  '  s$    
r/  c                 C   sh   |d u r|  d|S t|ddk rZt|}|d urZt|d| }| j dt|d}t| ||S )Nr   rz   r   rG   rH   )r   r   r  r   rJ   rK   _size_helperr   r$   r~   rankr   r   r   r  I  s    
r  c                 C   s|   ||kr|S t |}|d urTtt|}|| ||  ||< ||< | jd||dS t  rp| jd|d||dS tdd S )Nr  r  	transposeint)overload_nameZdim0_iZdim1_izAUnsupported: ONNX export of transpose for tensor of unknown rank.)r   r   r   r  r   r   r  rU   )r   r$   dim0dim1r9  r   r   r   r   r:  T  s    

r:  r   c                 C   s*   |t tdt|kr|S | jd||dS )Nr   r  r  )r   r  r?   r   )r   r$   r"  r   r   r   permutel  s    r?  c                 C   s   t | ||S r!   )r   )r   r$   r  r   r   r   views  s    r@  c                 C   s   |  d|}t| ||S r   r%   r&   r   r   r   view_asw  s    rA  c           	      C   s   |d u rt ddddS t ||}|d u r8t ddS || d | }|g||  }|| }|rl|| | jd||||dS )	Nunsafe_chunkr*   r+   'Dynamic number of outputs not supportedunknown dimension sizer,   Splitsplit_ir   r1  )r   r1   _get_tensor_dim_sizer4   r   r   )	r   r$   chunksr~   _outputsr  
split_sizesplitsleftoverr   r   r   rB  |  s    
rB  c           
      C   s   t ||st ddddS | d }| dkrDt| ||||S t |dd}t |dd	}t ||}|d u r|d ur|| }nt dddd
S |g||  }|| }	|	r||	 | j	d||||dS )Nsplitr*   r+   rC  rN   r   rz   rK  r~   z$Unknown dimension size not supportedrE  rF  )
r   _is_split_staticr1   r   r~   split_with_sizesr   rH  r   r   )
r   r$   split_size_or_sizesr~   rJ  Z	split_valrK  r  rL  rM  r   r   r   rN    s*    

rN  c                 C   s   t | ||||S r!   )rN  )r   r$   rQ  r~   rJ  r   r   r   unsafe_split  s    rR  c                 C   s0   t ||st ddddS | jd||||dS )NrP  r*   r+   rC  rE  rF  )r   rO  r1   r   r   r$   split_sizesr~   rJ  r   r   r   rP    s
    rP  c                 C   s   t | ||||S r!   )rP  rS  r   r   r   unsafe_split_with_sizes  s    rU  c                    s\   |d u rt ddddS jd|dg|  |d}|dkr@|gn|} fdd	|D }|S )
Nunbindr*   r+   rC  rE  r,   rF  c                    s   g | ]}t | gqS r   )r   _squeeze_helper)r   ra   r   r   r   r     s   zunbind.<locals>.<listcomp>)r   r1   r   )r   r$   r~   rJ  r1  Zsqueezed_outputsr   r   r   rV    s    rV  c                 C   sr   t |}t |s\|dk r\|dkr*d}n|d }t j| ||g|g|gd}t | ||gS | jd|||dS d S )Nr   r  r   r,   r   r   r   r%  r   )r   r3   r/   _slice_helperrW  r   )r   r$   r~   r   	end_indexZ
slice_noder   r   r   select  s    
r\  c                 C   s   |  d||S r:   r   rx   r   r   r   square  s    r]  c                 C   sH  |d u r|  d|S t|dd}|dk rt|}|d urxtdt| d d d t||  d	 d
  ||7 }ntddS t||}|d u rtdt| d d t| d d d d  tj	| ||gdS |dkrtdt| d d t| d d d d  |S tdt| d d  tj	| ||gdS )NSqueezerz   r~   r   z'ONNX export squeeze with negative axis - might cause the onnx model to be incorrect. (Negative axis is not supported in ONNX. Axis is converted to & based on input shape at export time. CPassing an tensor of different rank in execution will be incorrect.squeeze%negative axis with unknown input rankz5This model contains a squeeze operation on dimension z on an input z7with unknown shape. Note that if the size of dimension z of the input zVis not 1, the ONNX model will return an error. Opset version 11 supports squeezing on zMnon-singleton dimensions, it is recommended to export this model using opset zversion 11 or higher.r   r,   z. The size of z%this dimension in the given input is z. The model will zWbe exported without the squeeze node. If the model is intended to be used with dynamic z-input shapes, please use opset version 11 to zexport the model.z. If the model is z_intended to be used with dynamic input shapes, please use opset version 11 to export the model.)
r   r   r   r   r'  r(  strr4   rH  rW  )r   r$   r~   Zsqueeze_dimr9  dim_sizer   r   r   rd    s    




rd  c              	   C   s   t |}|d urT|dkr8t | |ttd|d }n|dkrTt | |dg}d}t |}|d ur|d ur||ksJ d||| d||S )Nr   r,   r   z0rank(x) should be >= rank(slope) but got {} < {}PRelu)r   r   r   r   r  r   r   )r   r$   r)  Z	self_rankZweight_rankr   r   r   prelu  s$    

rj  c                 C   s   |  d||  d|S )Nr;   r   r   r   r   r   r   silu3  s    rk  c                 C   s   |  d||  d|  d|S )Nr;   r   Softplusr   r   r   r   r   mish7  s    rm  c              	   O   s  | dd}| dd}t|}|d   }t|d  oP|du pPtj|k }|r|D ]>}	|	 rZ|	  |krZt	d| d| d|	   qZt
|D ]4\}
}	|	 rt|	s| jd	|	tj| d
||
< q| j|g|R i |}|r| jd	|tj| d
}|S )a  Some PyTorch operators (e.g., Clip/Min/ReLU/Pad) are super set of ONNX in terms of data types.
    This function maximizes the exportability of PyTorch-ONNX by allowing ONNX-unsupported PyTorch
    operator data type. For example, `Cast<int>(Clip<float>(Cast<float>(INPUT)))` can be used to mimic
    `Clip<int>(INPUT)` (opset version < 12).

    Args:
        g (torch._C.Graph): graph to write the ONNX representation into.
        op_name (str): operator name in ONNX.
        *args (tuple): operands to the operator.
        **kwargs (dict): attributes to the operator along with "opset_before" (optional, None by default)
            indicating the smallest opset version to trigger such casting behavior and "target_float_t"
            (optional, "Float" by default) indicating the data type of internal operator.

    Returns:
        Optional[torch._C.Value, Tuple[torch._C.Value, ...]]: output(s) of the operator.
    opset_beforeNtarget_float_tr\   r   z
Inputs of z must have same dtype. Got z and rX   rZ   )popr   r^   r_   r   r`   r	   export_onnx_opset_versionisCompleteTensorrU   	enumerater   r]   )r   op_namerB   r   rn  ro  inputsZdtype_0Zrequire_castr   rz   r$   r   r   r   op_with_optional_float_cast;  s0    rv  c                 C   s   t | d|ddS )NRelu   rn  rv  r   r   r   r   relul  s    r{  c                 C   s   t | d|dd}t| |dS )Nrw  rx  ry     )rv  	clamp_max)r   r   r{  r   r   r   relu6q  s    r~  c                 C   s   |  d|S )NCeilr   r   r   r   r   ceilw  s    r  c                 C   s   |  d|S )Nrc   r   r   r   r   r   rQ   {  s    rQ   c                 C   s.   t | || jdtdgd}t| |dgS NrG   r   rH   )r  r   rJ   r  r   rW  )r   r$   sz_0r   r   r   _len  s    r  c                 C   s@   t |dkrt ddS t |dkr4t ddS | d|S )Nr   	thresholdznon-zero thresholdznon-zero valuerw  )r   r2   r4   r   )r   r$   r  rN   r   r   r   r    s
    r  c                 C   s$   t |dd}| jd|t |dS )Nr   negative_slope	LeakyRelur   )r   r   r   r2   )r   r   r  inplacer   r   r   
leaky_relu  s    r  c                 C   sP   t ||}|d ur$|d dks$J | jd||dd\}}| d|| d|S )Nr   r   rE  )r   r1  r;   r   )r   rH  r   )r   r   r~   rh  firstsecondr   r   r   glu  s
    r  c              
   C   sZ  t |}|d ur|dk r"|| }||d k}|rptt|}|d ||  ||< |d< | jd||d}|d }| jd||d}|r|  dkrt |d	d
}| jd|t j| d}|r| jd||d}|S | d|| jd||gdd}| d|}	t j	| |	|gd}
| d|	|
}|rV|  dkrVt |d	d
}| jd|t j| d}|S )Nr   r,   r  r  r  SoftmaxrY  r
   rz   re   rX   rZ   r8   	ReduceMaxr   Exprf  rW   )
r   r   r   r  r   r   r   r   r   _reducesum_helper)r   r   r~   re   	input_dimis_transpose_requiredr   softmaxparsed_dtypeexpr   r   r   r   r    s:    
r  c                 C   s@   t |d}|dkr4| d| d| d|||S | d|S )NrE   r,   rW   rl  r;   )r   r  r   )r   r$   r   r  Z
beta_constr   r   r   softplus  s     r  c                    s   t | }|d ur$|t d  nd d u sBtdd D rNt tdS fddtdtD   fddtdt D   fddtdtD fddtdtD S )	Nc                 S   s   g | ]}|d u qS r!   r   r   rz   r   r   r   r         z)get_pool_ceil_padding.<locals>.<listcomp>input size not accessiblec              	      sB   g | ]:}t t | d |   |  t|  d qS r   r,   )r;  mathr  ru   r  )r~   kernel_sizepaddingstrider   r   r     s   0r   c                    sD   g | ]<} | d  |  | |  kr8 | d  n | qS r,   r   r  )ceiled_output_dimr~   r  r  r   r   r     s   "c                    sP   g | ]H}| d krdn2| | d|    | d  |  d    qS r,   r   r   r   r  )r  r~   r  r  r  r   r   r     s   
*c                    sd   g | ]\}| d |    | krT|  | d k rDt | q^t  | d n
t | qS r  r;  r  )r  r  padding_ceilr   r   r     s   )r   _get_tensor_sizesr?   anyr4   r   r  )r   r  r  r  sizesr   )r  r~   r  r  r  r  r   get_pool_ceil_padding  s"    


r  c              	      s>   t ddddddt dddddd fdd}|S )NTFrD   r   rz   c                    s2  t |dhkrt dS |s&|}t|}|rbt||||}|tdd t||D  }n|d }|||d}r| jd|fddi|\}	}
| jd|dd	d
 tD dd
 tD d\}}tj| |dd
 tD ddd}t	| |
|}
|	|
fS | jd|fddi|}	|	S d S )Nr,   dilationc                 s   s   | ]\}}|| V  qd S r!   r   r   ar$  r   r   r   	<genexpr>  r  z1_max_pool.<locals>.symbolic_fn.<locals>.<genexpr>r   )kernel_shape_ipads_i	strides_iMaxPoolr1  c                 S   s   g | ]}d qS r  r   r   _r   r   r   r   1  r  z2_max_pool.<locals>.symbolic_fn.<locals>.<listcomp>c                 S   s   g | ]}d qS r  r   r  r   r   r   r   2  r  )r1  r  r  c                 S   s   g | ]}d | qS )r   r   r  r   r   r   r   8  r  r   rX  )
setr   r4   tupler  zipr   r  rZ  r7   )r   r   r  r  r  r  	ceil_moder  r   rr*  r  Zflattened_indicesrP   r   ndimsreturn_indicestuple_fnr   r   symbolic_fn  sB    
z_max_pool.<locals>.symbolic_fn)r   quantized_argsr   )r   r  r  r  r  r   r  r   	_max_pool  s    4r  
max_pool1dr,   )r  
max_pool2dr   
max_pool3d   max_pool1d_with_indicesmax_pool2d_with_indicesmax_pool3d_with_indicesc                    sf   t dt dddddddd
tjttdf ttdf ttttdf f ttd fdd	}|S )NTrD   r   rz   r   .)r   r  r  r  r  count_include_padc                    s   |s|}t |||| }|}|rL| jd|d| d ddd}dt| }|r|t||||}	|tdd	 t|	|D  }n|d }| jd
||||d}
|
S )NPad)r   r   r   constantr   )r  mode_svalue_fr   c                 s   s   | ]\}}|| V  qd S r!   r   r  r   r   r   r    s   z1_avg_pool.<locals>.symbolic_fn.<locals>.<genexpr>AveragePool)r  r  r  )r   _avgpool_helperr   r?   r  r  r  )r   r   r  r  r  r  r  divisor_overrideZadjusted_paddingr  outputr   r  r   r   r  c  s:    

z_avg_pool.<locals>.symbolic_fn)N)r   r  r   r   Valuer   r;  r   )r   r  r  r   r  r   	_avg_poolb  s    	 

)r  
avg_pool1d
avg_pool2d
avg_pool3dc                    s"   t dd fdd}|S )NTFc              	      s  zt dW n ty,   t d Y S 0 dgt krTdkrT| d|S t |}z|dd   W n ty   d  Y n0  d u stdd  D rΈdgt kr| d	|d fS t d
S  fddt	dt D }|dgt| kr0dgt kr$| d	|d fS t dS  fddt	dt D }dkr|| |||dt  dt  dS | j|||d}|S )Nr   z4adaptive pooling, since output_size is not constant.r,   r  ZGlobalAveragePoolr   c                 S   s   g | ]}|d u qS r!   r   r  r   r   r   r     r  z7_adaptive_pool.<locals>.symbolic_fn.<locals>.<listcomp>ZGlobalMaxPoolr  c                    s   g | ]} | |  qS r   r   r  r~   output_sizer   r   r     r  r   z-output size that are not factor of input sizec                    s    g | ]}t  | |  qS r   r  r  r  r   r   r     r  r  r  r  F)r  r  )
r   
_parse_arg	Exceptionr  r?   r   r  r  r4   r  )r   r   r  r  rn   kr  r   r   r  r^   r  r   r    s:    



$z#_adaptive_pool.<locals>.symbolic_fn)r   r  )r   r^   r  r   r  r   r  r   _adaptive_pool  s    
*r  adaptive_avg_pool1dr  adaptive_avg_pool2dadaptive_avg_pool3dadaptive_max_pool1dr  adaptive_max_pool2dadaptive_max_pool3dc                 C   sT   t | tsJ t|d d  dg| d t|   }|dd d |dd d  }|S )Nr   r   r  )r   r;  r   r?   )r~   padpaddingsr   r   r   _prepare_onnx_paddings  s    &r  c              	   C   sd   t | d} t | r`t | r`t | }zdd |D } W n" ty^   t dddd Y S 0 | S )Nr   c                 S   s   g | ]}t |d dqS )rz   r  )r   r   )r   rD   r   r   r   r     s   z)_convert_padding_node.<locals>.<listcomp>r  r*   r+   z)The sizes of the padding must be constant)r   r  r/   r  r|   r  r1   )r  
input_listr   r   r   _convert_padding_node  s    


r  c              	   C   sh   d}zt |dd}W n" ty8   t dddd Y S 0 t|}tt ||}t| d||||ddS )	Nr  rE   rN   r  r*   r+   z*The value for the padding must be constant)r  r  r  rn  )r   r   r  r1   r  r  r   rv  )r   r   r  rN   r4  r  r   r   r   constant_pad_nd  s    
r  c              
   C   sF  t |}t|d dksJ t|d }|}t|D ]
}|d| d   }|d| d   }g }	|dkrtj| |d| g|d  gdgd}
|	|
 |dk s|dk rtj| |d| gtd| gdtd|   gd}|	| n
|	| |dkr$tj| |d| gdg|gd}|	| | jdg|	R dd| i}q4|S )Nr   r   r,   r  rX  r   r   )r  r?   r  r   rZ  r   maxr   )r   r   r  r  ndimcuridxpad_lpad_rr   leftmiddlerightr   r   r   _pad_circular  s<    



r  c                 C   s0   d}t |}tt||}t| d|||ddS )Nreflectr  r+   r  r  rn  r  r  r   r   rv  r   r   r  r4  r  r   r   r   reflection_pad;  s    r  c                 C   s0   d}t |}tt||}t| d|||ddS )Nedger  r+   r  r  r  r   r   r   replication_padD  s    r  c                 C   sp   t |d}|dkr t| ||S |dkr4t| ||S |dkrJt| |||S |dkr^t| ||S td| d S )NrP   	replicater  r  circularzUnrecognized padding mode )r   r  r  r  r  r  rU   )r   r   r  r4  rN   r   r   r   r  U  s    r  c                    s    fdd}|S )Nc                    s`   t | |\}}t  t |}|r6t dS |d u rNt | || }| jd||dS )Nzalign_corners == TrueUpsampler  )r   _get_interpolate_attributes_interpolate_warningr3   r4   _interpolate_size_to_scalesr   )r   r   r  rB   scalesalign_cornersr~   interpolate_moder   r   r   r  d  s    

z!_interpolate.<locals>.symbolic_fnr   )r   r~   r  r  r   r  r   _interpolatec  s    r  upsample_nearest1dnearestupsample_nearest2dr0  upsample_nearest3d   upsample_linear1dlinearupsample_bilinear2dupsample_trilinear3dc           	      C   s*   t | |||||\}}| jd|||dS )Nr  r  )r    _interpolate_get_scales_and_moder   )	r   r   r  scale_factorr4  r  recompute_scale_factor	antialiasr  r   r   r   __interpolate}  s    r  c                 C   s$   |   dkrtd| d|S NBoolzOONNX export does NOT support exporting bitwise Not for non-boolean input valuesrh   r^   r_   r   r   )r   inpr   r   r   bitwise_not  s
    r  c                    s    fdd}|S )Nc                    s    fdd}|S )Nc                    s   | j d | ||tj dS NrX   rZ   )r   r   r]   r   r   r'   r   to_typer   r   wrap_with_cast  s
    
zGwrap_logical_op_with_cast_to.<locals>.decorator.<locals>.wrap_with_castr   r   r  r  r   r   	decorator  s    z/wrap_logical_op_with_cast_to.<locals>.decoratorr   r  r   r   r  r   wrap_logical_op_with_cast_to  s    
r"  c                    s    fdd}|S )Nc                    s    fdd}|S )Nc                    s@   t  d }t|   }|| || |d|| |dS )N_cast_{}F)globalsr   r"  r^   r_   )r   r   r'   Zto_cast_funcZfrom_cast_funcr  r   r   r    s
    zPwrap_logical_op_with_cast_to_and_from.<locals>.decorator.<locals>.wrap_with_castr   r  r  r   r   r     s    z8wrap_logical_op_with_cast_to_and_from.<locals>.decoratorr   r!  r   r  r   %wrap_logical_op_with_cast_to_and_from  s    
r%  c                    s    fdd}|S )Nc                    s   |  d | ||S )Nrh   r   r  funcr   r   wrap_with_not  s    z4wrap_logical_op_with_negation.<locals>.wrap_with_notr   )r'  r(  r   r&  r   wrap_logical_op_with_negation  s    r)  c                 C   s$   |   dkrtd| d|S r  r  rx   r   r   r   __not_  s
    r*  c                 C   sH   t | tjr:t | tjr:| jdtjdtjddS | d||S )NrG   Trd   rH   ri   )r   r^   r   DeviceObjTyper   rJ   rK   boolr<   r   r   r   eq  s
    
r-  c                 C   s   t | ||S r!   )r-  r<   r   r   r   ne  s    r.  c                 C   s   t | ||S r!   gt_implr  r   r   r   gt  s    r1  c                 C   sz   |   d url|   dkrl|   d url|   dkrl| jd|tjd d}| jd|tjd d}| d||S )Nr  rX   IntrZ   Greaterr^   r_   r   r   r]   r  r   r   r   r0    s    r0  c                 C   s   t | ||S r!   lt_implr  r   r   r   lt  s    r7  c                 C   sz   |   d url|   dkrl|   d url|   dkrl| jd|tjd d}| jd|tjd d}| d||S )Nr  rX   r2  rZ   Lessr4  r  r   r   r   r6    s    r6  c                 C   s   t | ||S r!   r5  r  r   r   r   ge  s    r9  c                 C   s   t | ||S r!   r/  r  r   r   r   le  s    r:  c                 C   s:   |   dkr.|   dkr.| d||S tdd S )Nr  rg   zOONNX export does NOT support exporting bitwise AND for non-boolean input valuesr^   r_   r   r   r  r   r   r   __and_  s
     r<  c                 C   s:   |   dkr.|   dkr.| d||S tdd S )Nr  OrzNONNX export does NOT support exporting bitwise OR for non-boolean input valuesr;  r  r   r   r   __or_  s
     r>  c                 C   s:   |   dkr.|   dkr.| d||S tdd S )Nr  rf   zOONNX export does NOT support exporting bitwise XOR for non-boolean input valuesr;  r  r   r   r   __xor_  s
     r?  r  c                 C   s   |  d||S )Nrg   r   r  r   r   r   logical_and  s    r@  c                 C   s   |  d||S )Nr=  r   r  r   r   r   
logical_or  s    rA  c                 C   s   |  d||S )Nrf   r   r  r   r   r   logical_xor  s    rB  c                 C   s   |   |   kr6| jd|tj|    d}| jdtjdtjdd}t|sp| jd|tjd d}| d||}| jd|tj|    d}| d	||}|S )
NrX   rZ   rG   r   rd   rH   r\   PowrW   	r^   r_   r   r   r]   rJ   rK   float32r`   )r   r$   r'   twotwo_powrshiftr   r   r   	__rshift_!  s"    
rI  c                 C   s   |   |   kr6| jd|tj|    d}| jdtjdtjdd}t|sp| jd|tjd d}| d||}| jd|tj|    d}| d	||}|S )
NrX   rZ   rG   r   rd   rH   r\   rC  r;   rD  )r   r$   r'   rF  rG  lshiftr   r   r   	__lshift_9  s"    
rK  c              	   C   sh   |   dkr&| jd|tjd d}|d u rXt| |}t| || jdtdd|S | d|||S )Nr  rX   rZ   rG   r,   rH   Where)	r^   r_   r   r   r]   nonzero_unbind_helperrJ   rK   )r   	conditionr$   r'   rJ  r   r   r   r  Q  s    
r  c           	      C   s   t |}|d u rt ddS |dk r.|| }||d k}|r|tt|}|d ||  ||< |d< | jd||d}|d }| jd||d	}|r|  d
krt |dd}| jd|t j	| d}|r| jd||d}|S )Nr~   fONNX and PyTorch use different strategies to split the input. Input rank must be known at export time.r   r,   r  r  r  
LogSoftmaxrY  r
   rz   re   rX   rZ   )
r   r   r4   r   r  r   r   r   r   r   )	r   r   r~   re   r  r  r   Z	return_opr  r   r   r   log_softmax`  s.    
rR  c                 C   s"  t |}z|dd  }W n ty0   d }Y n0 |d u sLtdd |D rTtd||g}t |s~t |dkr~|| |dd  ||| ||	d}tdd |D r|sJ t|t|ksJ ||d	< | j	|rd
ndg|R i |}t |st |dkr| 	d||S |S d S )Nr   c                 S   s   g | ]}|d u qS r!   r   r  r   r   r   r     r  z _convolution.<locals>.<listcomp>zDUnsupported: ONNX export of convolution for kernel of unknown shape.r,   )r  r  r  Zdilations_iZgroup_ic                 s   s   | ]}|d kV  qdS r   Nr   )r   or   r   r   r    r  z_convolution.<locals>.<genexpr>Zoutput_padding_iZConvTransposeConvr)   )
r   r  r  r  rU   r  r   r   r?   r   )r   r   r)  biasr  r  r  
transposedoutput_paddinggroups	benchmarkdeterministiccudnn_enabled
allow_tf32weight_sizekernel_shaperB   r   r   r   r   r   _convolution  s@    




 r`  c                 C   s"   t | ||||||dd|d d d d S NFr   r`  r   r   r)  rV  r  r  r  rY  r   r   r   conv1d  s     rd  c                 C   s"   t | ||||||dd|d d d d S ra  rb  rc  r   r   r   conv2d  s     re  c                 C   s"   t | ||||||dd|d d d d S ra  rb  rc  r   r   r   conv3d  s     rf  c	           	      C   s"   t | ||||||d||d d d d S NTrb  	r   r   r)  rV  r  r  rX  rY  r  r   r   r   conv_transpose1d  s     ri  c	           	      C   s"   t | ||||||d||d d d d S rg  rb  rh  r   r   r   conv_transpose2d  s     rj  c	           	      C   s"   t | ||||||d||d d d d S rg  rb  rh  r   r   r   conv_transpose3d+  s     rk  c
                 C   s   t |d t rBt |||||gsBtjdk rBt ddddS t | |||||\}}}}| j	d||||||d| |s~dndd	}
|s|
S |
\}}}}}|
|  |
|  |d	|   |d	|   |S d S )
N
batch_norm   BatchNormalizationr*   zaAll input tensors must have the same `dtype`. Turn off Autocast or export using opset version 15.r,   r
  )	epsilon_fZ
momentum_fr1  zbatch_norm_dead_output-)r   check_training_moderJ   is_autocast_enabledargs_have_same_dtyper	   rq  r1   _batchnorm_helperr   r   r^   setDebugName	debugName)r   r   r)  rV  running_meanrunning_vartrainingmomentumepsr\  ra   resZnew_running_meanZnew_running_varZ
saved_meanZ	saved_varr   r   r   rl  A  sJ    
rl  c              	   C   s   t  r | jd||||||dS dd tt|ddD }t | d}t | |}	| jd||d	}
t| ||
}| jdt| |||d	}t	| t
| ||	}| d
||}|d u st |st| ||}|d u st |st
| ||}|S )N
layer_norm)Znormalized_shape_ieps_fZcudnn_enable_ic                 S   s   g | ]
}| qS r   r   r  r   r   r   r     r  zlayer_norm.<locals>.<listcomp>r   r         @r   rf  rW   )r   r   r  r  r?   _generate_wrapped_numberr   r7   powr   r-   r  r=   )r   r   normalized_shaper)  rV  rz  Zcudnn_enabler   Ztwo_cstZeps_cstr   	numeratorvariancedenominatorr|  r   r   r   r|  x  s.    
r|  c
                 C   s.  t |d t |d}
|d u s*t |rn|
d u r:tdtdg|
 d|   d }| j	d|d}|d u st |r|
d u rtdtd	g|
 d|   d }| j	d|d}|d u st |s|d u st | r| j	d
||||dS t 
|}| }|d }|d u r*td|d }d|d< || |d< t| || j	dtj|gtjdd}t| || j	dtj|gtjdd}t| || j	dtj|gtjdd}t| || j	dtj|gtjdd}| 	d|| j	dt|d}t| |||||||||	
}t| || j	dt|dS d S )Ninstance_normr,   zCUnsupported: ONNX export of instance_norm for unknown channel size.rF   torch.r  rG   rH   r   InstanceNormalizationro  r   zJUnsupported: ONNX export of instance_norm training for unknown batch size.rd   Reshape)r   rp  rH  r  rU   rJ   rK   r^   r_   r   r  copyrepeatrj   r  rl  r@  )r   r   r)  rV  rv  rw  use_input_statsry  rz  r\  channel_sizeweight_value
bias_value
input_sizeZinput_size_reshaper   cweight_bias_Zrunning_mean_Zrunning_var_input_reshapedra   r   r   r   r    s    

r  c                    s   t  rjd ||dS t }z|  }W n tyH   d }Y n0 |d urtd||}t||d |} fddt||D }	t|}
ttd|
	
   fdd|	D }jdg|R d	 iS t d
dS d S )Nunfold)Zdimension_isize_iZstep_ir   r,   c              	      s*   g | ]"\}}t j g|g|gd qS )rX  r   rZ  )r   lowhi)	dimensionr   r   r   r   r     s   zunfold.<locals>.<listcomp>c              
      s(   g | ] }t jd |d gqS )r  r  )r   r   r   r   )r  r   permr   r   r   
  s   r   r   Unfoldr  )r   r   r  r  r  r  r  r?   r   r   rp  r   r4   )r   r   r  r  stepr  ZsizedimZlow_indicesZ
hi_indicesr   r  r  r   )r  r   r   r  r   r    s*    

r  c                 C   sF   |r|dkrt ddS |r0|dkr0t ddS | jd|t |dS )NrF   r   zdoes not support scale in Eluinput_scalez#does not support input_scale in EluElur  )r   r4   r   r2   )r   r   r6   r   r  r   r   r   elu  s    r  c                 C   s   |  d|S )NSelur   r   r   r   r   selu!  s    r  c                 C   s   t | |||S r!   )r   _select_helper)r   r$   r~   r   r   r   r   index_select%  s    r  c                 C   s   t |rt |}n|g}t  rH|g| ||g }| jdg|R  S t |d}t|dkrv|rpt| ||S |S nt ddd d S )N	index_putr$  r   r*   r+   )	r   r  r|   r   r  r  r?   r-   r  )r   r$   indices_list_valuevalues
accumulateindices_listrB   r   r   r   r  -  s    
r  c           	      C   st   t |d}t  r*| jd|||d|dS t | |||\}}t |}t | ||}t| ||d }t| ||||S )Nrz   
index_fill
int_Scalar)r<  r   )	r   r  r   r  _index_fill_reshape_helperr3   r   r  scatter)	r   r$   r~   r   rN   	dim_valueexpanded_index_shapeexpanded_indexexpanded_valuer   r   r   r  A  s"    	
r  c                 C   sL   t |d}t  r(| jd||||dS t | |||\}}t| ||||S )Nrz   
index_copyr   )r   r  r   r  r  r  )r   r$   r~   r   sourcer  r  r  r   r   r   r  W  s    r  c                 C   s   t jj}|rt jj}| jd| d|| d|dd}ttdt|d }t	| t
| |||d }|rxt| ||}	nt| ||}	| jd|	|d}
tj| |
dgddS )	Nr   r   r   rY  r,   rX   rZ   r   )_C_onnxTensorProtoDataTyper  INT32r   r   r  r   r   r  r   r9  r1  r  )r   r$   
boundaries	out_int32r  out_type	new_shapeZunsqueeze_axesZexpanded_boundariescondcond_outr   r   r   	bucketizea  s     "
r  c                 C   sh   t |}t |}||kr(|d ur(|S |d urF| jd|t j| dS t  r\| d||S tdd S )NrX   rZ   type_aszUnsupported: ONNX export of type_as for tensor of unknown dtype. Please check if the dtype of the parameter passed to the type_as function is correct.)r   r   r   r]   r   r  rU   )r   r$   r'   r   other_dtyper   r   r   r    s    

r  c           	      C   s   t  r| jd||||dS t j| t| |||gdd}t j| t| |||gdd}t j| t| |||gdd}t| t| t| ||| jdt	|gd}t
| ||S )Ncosine_similarity)r   r}  r   r   rG   rH   )r   r   r  r  r=   r  r   r   rJ   rK   rC   )	r   x1x2r~   rz  crossZx1_l2Zx2_l2Zdiv_tensr   r   r   r    s    &r  c                 C   s   t |s | jdt|gd}t| | jdtjdgtjddt| ||}t j| t	| t
| |||dgt |dd}t	| ||S )NrG   rH   r,   rd   r  rz   r   )r   r/   r   rJ   rK   rC   ru   r-   r  r  r7   r  )r   input1input2prz  r   Zinv_pZ	summationr   r   r   pairwise_distance  s    


r  c                 C   s   |S r!   r   )r   r   Zunused_memory_formatr   r   r   clone  s    r  c                 C   s   |  d|S )NAbsr   rx   r   r   r   abs  s    r  c                 C   s   |  d|S )NLogr   rx   r   r   r   log  s    r  c              
   C   s"   t | t| t| td||S )Nr,   )r  r-   r   r   rJ   r   rx   r   r   r   log1p  s    r  c              	   C   s*   d}|  dt| || j dt|gdS )NgUk@rW   rG   rH   r   r  rJ   rK   )r   r$   Z_ln10r   r   r   log10  s    r  c                 C   sf   |    }}t|s4d}| jd|tj| d}t|sT| jd|tj| d}| d||}|S )Nr\   rX   rZ   rC  )r^   r_   r   r`   r   r]   )r   r$   exponentZf_dtyper   r  r   r   r   r    s    

r  c              	   C   s|   t |rt| ||S t |r,t| ||S t |rdt |rdt| d|t |dt |dddS t| t| |||S d S )NCliprE      min_fmax_frn  )r   r  r}  	clamp_min_is_constantrv  r  )r   r$   minr  r   r   r   clamp  s    



	r  c                 C   s\   t |r$t| d|t |dddS |  }| jd|t j| d}t| d||ddS d S )	Nr  rE   r  )r  rn  rX   rZ   Maxry  r   r  rv  r  r^   r_   r   r]   )r   r$   r  re   r   r   r   r    s    
r  c                 C   s\   t |r$t| d|t |dddS |  }| jd|t j| d}t| d||ddS d S )	Nr  rE   r  )r  rn  rX   rZ   Minry  r  )r   r$   r  re   r   r   r   r}   	  s    
r}  c                 C   s   |d u r |d u r | j d|ddS |d u r:t| d||ddS t|dd}t|dd	}| j d||g|d
}| j d|||d}||fS d S )Nr  r   r   r  r  ry  rz   r~   r   r   ArgMaxr   r   r   rv  r   r   )r   r$   dim_or_yr   r~   r  r*  r   r   r   r  	  s    r  c                 C   s   t | ||dS N)r  )r  r  r   r   r   maximum	  s    r  c                 C   s   |d u r |d u r | j d|ddS |d u r:t| d||ddS t|dd}t|dd	}| j d||g|d
}| j d|||d}||fS d S )N	ReduceMinr   r  r  r  ry  rz   r~   r   r   ArgMinr  r  )r   r$   r  r   r~   r  r*  r   r   r   r  "	  s    r  c                 C   s   t | ||dS r  )r  r  r   r   r   minimum2	  s    r  c                 C   s   | j d|||dS )Nr  r   r   r   r$   r~   r   r   r   r   amax6	  s    r  c                 C   s   | j d|||dS )Nr  r   r   r  r   r   r   amin;	  s    r  c                 C   sR   d|i}t |s*t |dd}|g|d< | jd|fi || jd|fi |fS )Nr   rz   r~   r   r  r  )r   r  r   r   )r   r$   r~   r   Zreduce_kwargsr   r   r   aminmax@	  s    

r  c                 C   s   |  d|S )Nr  r   rx   r   r   r   r  L	  s    r  c                 C   s8   t |d |s|S td | jd||dd\}}|S )NdropoutzDropout is a training op and should not be exported in inference mode. For inference, make sure to call eval() on the model and to export it with param training=False.Dropoutr   )Zratio_fr1  )r   rp  r'  r(  r   )r   r   r  trainr  r  r   r   r   r  P	  s    r  c                    s   t ddd fdd}|S )NrD   rE   rz   c                    s   |rt  dS |S )Nztraining mode)r   r4   )r   r   r  r  r   r   r   feature_dropout_	  s    z-_unsupported_dropout.<locals>.feature_dropoutr   )r   r  r   r  r   _unsupported_dropout^	  s    r  r  alpha_dropoutfeature_alpha_dropoutc                 C   s<   |dkrt d}n|dkr$t d}ntd|| |||dS )Nr,   ReduceL1r   ReduceL2z)ONNX export only p-norms with p of 1 or 2)r~   r   )r   rU   )r   r$   r  r~   r   rE   r   r   r   normt	  s    

r  c              	   C   sx   t  r| jd||||dS | jd|g dd}| jd|g dd}t| |||dg|gdgd}| jd|g ddS d S )	Nconv_tbc)Zpad_ir  )r,   r   r   r  )r   r,   r   r,   )r   r   r,   )r   r   r  r   rd  )r   r   r)  rV  r  convr   r   r   r  	  s    r  c                 C   s*   t  r| jd|||ddS t dS d S )N_uniquer   )sorted_ireturn_inverse_ir1  )r   r   r  r  )r   r   sortedreturn_inverser   r   r   r  	  s    r  c                 C   s0   t  r| jd||||ddS t ddd d S )N_unique2r  )r  r  Zreturn_counts_ir1  r*   r+   )r   r   r  r  )r   r   r  r  return_countsr   r   r   r  	  s    	r  r#  c                 C   s   t | |||||S r!   )zeros)r   r  re   layoutdevice
pin_memorymemory_formatr   r   r   empty	  s    r  c                 C   s   t | |||||S r!   )
zeros_like)r   r   re   r   r  r  r  r   r   r   
empty_like	  s    r  c                 C   sB   t |}|d u r0|d ur0|}t jt j| }t| |||||S r!   )r   r   r   r   r]   r  r   r$   r  re   r   r  r  r   r   r   r   	new_empty	  s    
r  c                 G   s8   t |dd}|d u rt jj}| jd|t j| d}|S )Nrz   re   rX   rZ   r   r   r  FLOATr   r   )r   scalarre   optionsr   r   r   scalar_tensor	  s
    r  c                 C   s,  t |dd}t |r|d u rHt |d   }t jt j| }t	 }t |D ]H}| j
dtdgd}t | ||}| j
d|t j| d}|| qX| j
d	g|R d
diS |d u r|  }t jt j| }t |rt |st |r| j
d|ddd}| j
d|t j| dS )Nrz   re   r   rG   r,   rH   rX   rZ   r   r   ConcatFromSequence)r   
new_axis_i)r   r   r  r|   r^   r_   r   r   r]   r   r   rJ   r  r"   r   _is_listr0   _is_scalar_list)r   datare   r  requires_gradr  r   Zshape_referencer   r   r   rK   	  s4    
rK   c                 C   s   t | |||S r!   )rK   )r   r  re   r  r   r   r   	as_tensor	  s    r  c                 C   sp   |d u rt jj}t |d}t|trNt|dkrN| jdt	g 
tjd}| jd|tj	dgt j| ddS )Nr   r   rG   rH   ConstantOfShaperd   r   r  r
  r  r   r   r?   r   rJ   rK   r  rj   r   r   r  re   r   r  r  sizes_r   r   r   r  	  s    r  c                 C   s>   |  d|}|d u rtjj}| j d|tjdgtj| ddS )Nr   r  r   rd   rH   r   r   r  r
  rJ   rK   r   r   r   re   r   r  r  r  r   r   r   r   r  
  s    r  c                 C   sB   t |}|d u r0|d ur0|}t jt j| }t| |||||S r!   )r   r   r   r   r]   r  r  r   r   r   	new_zeros
  s    
r  c                 C   sp   |d u rt jj}t |d}t|trNt|dkrN| jdt	g 
tjd}| jd|tj	dgt j| ddS )Nr   r   rG   rH   r  r,   rd   r  r  r   r   r   r    
  s    r   c                 C   s>   |  d|}|d u rtjj}| j d|tjdgtj| ddS )Nr   r  r,   rd   rH   r  r  r   r   r   r  0
  s    r  c                 C   sB   t |}|d u r0|d ur0|}t jt j| }t| |||||S r!   )r   r   r   r   r]   r   r  r   r   r   new_ones@
  s    
r  c           
   	   C   s   t |d}t |rX|d u r&t jjn|}t| ||||}t| ||| jdt	ddS t 
|dd}|d u rvt jjn|}t |d}	t|	trt|	dkr| jdt	g tjd}| jd	||dt j| dS d S )
Nr   rG   r,   rH   rz   re   r   r   r  )r   r  r/   r  r
  r  r-   r   rJ   rK   r   r   r   r?   r  rj   r@  r   )
r   r  rN   re   r   r  r  const_valuetmpr  r   r   r   fullJ
  s"    

r  c           
   	   C   s   t |d}t |dd}|d u r*t jjn|}t |r|t| ||||}| jd|t j| d}t	| ||| jdt
ddS | d	|}	| jd
|	t
|gt j| dS d S )NrE   rz   re   rX   rZ   rG   r,   rH   r   r  )r   r  r   r  r
  r/   r  r   r   r-   rJ   rK   r  r   )
r   r   
fill_valuere   r   r  r  r  r  r   r   r   r   	full_like_
  s"    

r!  c           	      C   sD   t |}|d u r0|d ur0|}t jt j| }t| ||||||S r!   )r   r   r   r   r]   r  )	r   r$   r  r   re   r   r  r  r   r   r   r   new_full}
  s    
r"  c                 G   s   t |dkrX|\}}}}}t| |dg}| jd||dd}t| ||||}	| d|	S t |dkr|\}}
}}}}| jdt| |dgt| |
dgdd}t| ||||}	| d|	S tdd S )Nr
  r   r   rY  ZEyeLiker|  zUnknown aten::eye signature)r?   r   r   r   r  r   )r   rB   r   re   r   r  r  rh  r   rK   mr   r   r   eye
  s"    r$  c                 G   s  t |dkrh|\}}}}t|d}|dkr6td|  dkoT|  dk}|  dkot|  dk}|  dk}	|  dk}
|s|	r|s|
r|  dkrtjt	j
jkrtdnBt| |d	g}t| |d	g}t| |d	g}| d
||||S nR|rd	n
t|d}|r4dn
t|d}t|d}tj| ||g|g|gdS nt |dkr|\}}}d	}|  dko|  dk}|  dko|  dk}|rd	n
t|d}|rdn
t|d}tj| ||g|g|gdS tdd S )Nr0  rz   r,   z"step!=1 is currently not supportedr
   NoneTyper   zUnsupported: ONNX export of Slice with dynamic inputs. DynamicSlice is a deprecated experimental op. Please use statically allocated variables or export to a higher opset version.r   ZDynamicSlicer   rX  r  zUnknown aten::slice signature)r?   r   r  rU   r   r   r^   r	   operator_export_typer  OperatorExportTypesONNXr   r   rZ  r   )r   r$   rB   r~   startendr  Zis_start_noneZis_end_noneZis_start_onnx_constZis_end_onnx_constZstart_unsqueezedZend_unsqueezedZdim_unsqueezedr   r   r   slice
  sx    

 r+  c                 C   s   t | d|||ddS )Nr  r  r  rz  )r   r$   min_valmax_valr   r   r   hardtanh
  s    r.  c                 C   s   t | |}| d||S r:   )hardsigmoidr   )r   r$   hsr   r   r   	hardswish
  s    
r1  c                 C   s   | j d|ddS )NHardSigmoidgUUUUUU?r  r   rx   r   r   r   r/  
  s    r/  c                 C   s   |  d|t| |S )Nr8   )r   r   rx   r   r   r   
tanhshrink
  s    r3  c              
   C   sZ   | j dt|gd}t| t| ||t| |t| |}|  d||| j dtdgdS NrG   rH   rL  r   )r   rJ   FloatTensorrA  r1  r7  r   )r   r$   lambdlambd_opr  r   r   r   
hardshrink   s    "r8  c              
   C   s   | j dt|gd}t| ||}|  d|t| ||| j dtdgd}t| |t| |}|  d|t| ||| j dtdgd}t| ||S r4  )r   rJ   r5  r1  r7   r7  r   r-   )r   r$   r6  r7  Zgt_condZgt_outZlt_condZlt_outr   r   r   
softshrink  s     

r9  c                 C   s   |S r!   r   rx   r   r   r   alias  s    r:  c                 C   s|   |dk rjt |}|d ur^tdt| d d d t|| d  d d  || d }nt d	d
S t j| ||gdS )Nr   z)ONNX export unsqueeze with negative axis r_  r`  ra  r,   rb  rc  r  re  rf  )r   r   r'  r(  rg  r4   r   r8  r   r   r   r    s2    

r  c                 C   sj   |d urt dd t |}z|| }W n ty@   d }Y n0 |d u rVt ddS | jd|||ddS )NSortz'Out parameter is not supported for sortr  TopKr   k_ir   r1  )r   r4   r  r  r   )r   r$   r~   	decendingra   Z
self_sizesrh  r   r   r   sort8  s    

r@  c                 C   s   |  d|}| j d|ddS )Nr   r   r   r  r   r#   r   r   r   numelJ  s    rA  c                 C   s8   |d urt dd |s$t dd | jd|||ddS )Nr<  z'Out parameter is not supported for topkzAscending TopK is not supportedr   r=  )r   r4   r   )r   r$   r  r~   largestr  ra   r   r   r   topkO  s    rC  c                 G   s  dd }||r|S t |dkr|d }t|d r|d   dkr|d  d }t|tjrt |jdkr|	 }t
|}n|}t|st|tjr|d   }| jd|tj| dS | jd|tj| dS nt |d	krt|d
 dd}| jd|tj| dS t |dkrJt|d dd}| jd|tj| dS t |dkrt|d dd}| jd|tj| dS tdS d S )Nc                 S   s   t | dkrL| d   dkpJ| d  tj pJt| d  tj	S t | dkrrt
| d dd}|d u S t | dv rt
| d dd}|d u S d	S )
Nr0  r   prim::devicer
  r,   rz   re   )r|     F)r?   r   r   r^   isSubtypeOfr   ListTypeofIntsr   r+  r   r   )rB   re   r   r   r   is_aten_to_device_only\  s    z"to.<locals>.is_aten_to_device_onlyr0  r   r   rN   rX   rZ   r
  r,   rz   re   r|  rE  zUnknown aten::to signature)r?   r   r/   r   r   r   rJ   r  r   itemr;  r^   r_   r   r]   r   r   r  )r   r$   rB   rI  re   tvalr   r   r   r  [  s:    
r  c                 C   s0   t jj}t| ||}| d||}| d||S )Nr  Tile)r   r  r  r  r   )r   r$   repeatsre   shape_r   r   r   r    s    r  c              
   C   s  |}t |r4t | || jdtdgd}d}n
t |}t |}t |}t |}|d u rlt	d|d u r|t	d|d u rt	d|
 }	t|D ] \}
}|d u rd\||
< |	|
< q|dks|d	krD|d d	krDt |s| jdt|d}|| dkrt d
dddS || }t| || jdt|gdd }nr|d	kr|| dkrlt d
dddS |d d u rt d
dddS |d || ksJ d|d }nt	dt }t | ||d}t | |||}d\||< |	|< t|D ]\}
}t| ||
 |d	 }| jdt|	d |d	  d|| jdt|	|d	 d  dg}| jdg|R ddi}t| ||d }t j| || jdt|ddd}|| q| jdg|R d|iS )NrG   r  rH   r   zGUnsupported: ONNX export of repeat_interleave for unknown repeats rank.zGUnsupported: ONNX export of repeat_interleave for unknown repeats size.zEUnsupported: ONNX export of repeat_interleave for unknown input size.)r   r  r,   repeat_interleaver*      z3Unsupported along dimension with unknown input sizez*Unsupported for cases with dynamic repeatsz2repeats must have the same size as input along dimz%repeats must be 0-dim or 1-dim tensor)r  r,   r   r   	allowzero)r   r  r"   r   rJ   rK   r3   r   r  rU   r  rs  
_is_tensorr  r1   r  r   _repeat_interleave_split_helperr  r   )r   r$   rM  r~   r  r   Zrepeats_dimZrepeats_sizesinput_sizesZinput_sizes_tempr  r  repsZfinal_splitsZr_splitsZi_splitsZr_splitZi_splitZr_concatr   r   r   rO    s    




 


rO  c           	      C   s  t |}t|dkr"t ddS tdd |dd  D rt j| t | |ddg| jd	t	d
d||d
d
gdd
d}| jd|g dd}t j| || jd	t	g ddd
d}t j| || jd	t	g ddd
d}t 
| |ddgS |d | | }t j| || jd	t	d||||d |d gdd
d}| jd|g dd}t j| || jd	t	d||d | |d | gdd
dS d S )Nr0  pixel_shuffleonly support 4d inputc                 s   s   | ]}|d u V  qd S r!   r   r  r   r   r   r  
  r  z pixel_shuffle.<locals>.<genexpr>r,   r   r  rG   r   r  rH   rQ  r  )r   r,   r0  r   r
  r  r  )r   r   r  r,   r   r   )r   r   r   r   r  r,   r
  r   r  r?   r4   r  r"   r   r   rJ   rK   rW  )	r   r$   upscale_factorr"  
after_viewafter_transpose	reshape_h	reshape_woutput_channelr   r   r   rW    sz    
	

rW  c           
      C   s  t |}t|dkr"t ddS tdd |dd  D rt j| t | |dg| jdt	d	d	d
|d	gdd	d}t j| || jdt	d	d	d	d	d
|gdd	d}| jd|g dd}t j| || jdt	g ddd	d}t 
| |ddgS |d | | }t j| || jdt	d
|d |d | ||d | |gdd	d}	| jd|	g dd}t j| || jdt	d
||d | |d | gdd	dS d S )Nr0  rW  rX  c                 s   s   | ]}|d u V  qd S r!   r   r  r   r   r   r  O  r  z"pixel_unshuffle.<locals>.<genexpr>r,   r  rG   r   r  rH   rQ  r  )r   r,   r  r
  r   r0  r  )r   r  r,   r,   r   r   r   rY  )
r   r$   downscale_factorr"  r]  r^  r\  Zfinal_reshaper_  r[  r   r   r   pixel_unshuffleJ  st    




ra  c           *   
      s  t d d d d d  g d}ttdd |D |}|rFd	nd
dkrvt  d|	  krvtddS t  d|	  ksJ  fddtdt D |
rʈjd|g dd}|r|rtddS 	dr|d	d  
  }d d }t|dd u r,tddS |	 }|}g }dksNdkrT|}ndkrj|\}}g }|d u r|tn|}dkrg dndkrg ddd fdd}fdd}fd d!}tD ]\}|r8d	kr||\}}}n||\}}t}||d f}nd	kr~|d
| \}} }!|d
| d \}"}#}$jd"|!|$dd#}n,|d
| \}} |d
| d \}"}#t}jd"||"dd#}jd"| |#dd#}d
| d
| d
 f}|||||g}%|%||g|R   dkr&|%||g|R   |r0i nd$d%i}&dkr~|	rR||g}'n|g}'jdg|%R d
|'d&|&\}}(n^dkrjdg|%R d
dd'|&\}}(n.dkr܈jdg|%R d(d)|&\}}(})|	rjd|g d*d}tj|jd+tg d,d-dd.}nt|dg}||( dkr||) q|
rjjd|g dd}dkrx|(njd"g|R d/di}dksdkr||fS dkrdkr|)njd"g|R d/di}|||fS d S )0NzVExporting a model to ONNX with a batch_size other than 1, with a variable length with z can cause an error z9when running the ONNX model with a different batch size. z4Make sure to save the model with a batch size of 1, z=or define the initial states (h0/c0) as inputs of the model. )rw  r   r   ZAffiner  ZThresholdedReluZ
ScaledTanhr2  r  Softsignrl  c                 S   s   g | ]}|  qS r   )lower)r   Zact_funr   r   r   r     r  z _generic_rnn.<locals>.<listcomp>r0  r   LSTMr,   zLSTMs with projectionsc                    s   g | ]} ||  qS r   r   r  )all_weightsweights_per_layerr   r   r     s   r   r  r  r  zRNN/GRU/LSTMzdropout in training modeRNNzunknown hidden sizeGRU))r,   r   r   r,   )r   r  )ri  )r  r0  )r,   r  c                    s.    fdd|D } j dg|R ddiS )Nc              	      s2   g | ]*\}}t j d g| g| gdqS )r   rX  r  )r   xyr   r   wr   r   r     s   z8_generic_rnn.<locals>.reform_weights.<locals>.<listcomp>r   r   r   r   )r   rm  r   	intervalsslicesr   rl  r   reform_weights  s    z$_generic_rnn.<locals>.reform_weightsc                    s`   |  }dkr|\}}n,dks*dkrF fdd|D \}}t  fdd||fD S )Nrg  rh  rd  c                    s   g | ]} |qS r   r   r   rm  r   hidden_sizereform_permutationrp  r   r   r     s   zC_generic_rnn.<locals>.transform_weights_no_bias.<locals>.<listcomp>c                 3   s   | ]}t  |d gV  qdS rS  r   r   rj  r   r   r   r    s   zB_generic_rnn.<locals>.transform_weights_no_bias.<locals>.<genexpr>)r  )layer_indexweights	weight_ih	weight_hhr   rs  layer_weightsrt  rp  variantr   r   transform_weights_no_bias  s    

z/_generic_rnn.<locals>.transform_weights_no_biasc                    s|   |  }dkr|\}}}}n0dks.dkrN fdd|D \}}}} j d||dd}t fd	d
|||fD S )Nrg  rh  rd  c                    s   g | ]} |qS r   r   rq  rr  r   r   r     s   z;_generic_rnn.<locals>.transform_weights.<locals>.<listcomp>r   r   rY  c                 3   s   | ]}t  |d gV  qdS rS  r   ru  rv  r   r   r    s   z:_generic_rnn.<locals>.transform_weights.<locals>.<genexpr>)r   r  )rw  rx  ry  rz  bias_ihbias_hhbias_concatr{  r   r   transform_weights  s    z'_generic_rnn.<locals>.transform_weightsc                    s&   dkr| S t j | dg|g|gdS )Nr,   r   rX  r  )rj  r)  r*  )r   
num_layersr   r   retrieve_state  s    z$_generic_rnn.<locals>.retrieve_stater   rY  direction_sbidirectional)r1  hidden_size_iZactivations_s)r1  r  Zlinear_before_reset_ir  )r1  r  )r   r   r,   r  rG   )r   r   r  rH   rQ  r   )r'  r(  dictr  r?   r   r4   r  r   
startswithrc  rH  r   r   r"   rJ   r  rW  )*r   r}  r   initial_statesre  
has_biasesr  r  r  r  batch_firstbatch_sizesZonnxActivationsZvariantToOnnxActivationMapnonlinearityw_hhunidirectionalprev_outputh_outsh0c0c_outsZsequence_lensr~  r  r  rz   ry  rz  r  Zstate_indicesZweight_ih_fZweight_hh_fbias_fZweight_ih_bZweight_hh_bZbias_bru  extra_kwargs
activationZh_outZc_outr   )	re  r   rs  r|  r  rt  rp  r}  rf  r   _generic_rnn  s   





	








&
&r  c
                 C   s2   t |t | }
}t| d||
|||||||	S )Nrd  r   r|   r  )r   r   hidden_vweight_vr  r  r  r  r  r  hiddenr)  r   r   r   
_lstm_fulll  s$    r  c
                 C   s4   t |t | }
}t| d||
||||||	|dS )Nrd  r  r  )r   r   r  r  r  r  r  r  r  r  r  r)  r   r   r   _lstm_packed  s$    r  c                 G   s2   t |d rt| g|R  S t| g|R  S d S Nr  )r   r0   r  r  r   rB   r   r   r   lstm  s    r  c                    s   t  |dg}t |} fdd|D }t |rB||||fn||f}t |rXdnd}	t d||||	dddddd\}
}}t  |dgt  |dgfS )	Nr   c                    s   g | ]}t  |d gqS r  r   ru  rv  r   r   r     r  zlstm_cell.<locals>.<listcomp>TFrd  r,   )r  r  r  r  r  )r   r   r|   rS  r  rW  )r   r$   r  w_ihr  b_ihb_hhr   r)  r  r  r  r  r   rv  r   	lstm_cell  s0    
r  c                    s^   t ddddddddd	fdd t ddddddddd	fdd fdd	}|S )
NrD   rz   rE   c
                    s&   t |}
t|  |||
||||||	S r!   r  )r   r   r  r  r  r  r  r  r  r  r)  r   r   r   	_rnn_full  s    
z"_one_hidden_rnn.<locals>._rnn_fullc
                    s(   t |}
t|  |||
|||||	|dS )Nr  r  )r   r   r  r  r  r  r  r  r  r  r)  r  r   r   _rnn_packed  s    
z$_one_hidden_rnn.<locals>._rnn_packedc                    s2   t |d r| g|R  S  | g|R  S d S r  )r   r0   r  )r  r  r   r   r     s    z!_one_hidden_rnn.<locals>.symbolicr   )r   r   r   )r  r  r   r   _one_hidden_rnn  s    r  rh  RNN_TANHRNN_RELUc                 C   sX   |  d|}| j d|| j dt|ddd}t rB|  d|S t| |dd d d S d S )	Nr   r%  rG   rH   r   rY  z_caffe2::Ranger0  )r   rJ   rK   r   r   arange)r   liker~   
like_shapestopr   r   r   _dim_arange  s    r  c                 C   s   |S r!   r   r   r   r   r   detach  s    r  c                 C   s   |dkrt d|S )Nr   z-onnx memory_format support is not implemented)rU   )r   r   r  r   r   r   
contiguous$  s    r  c                 C   sd   |r| j d|g dd}| tjj s6td|  dkrRt	| |d}| j d||dd	S )
Nr  r  r  z(Lengths must be a Tensor for ONNX exportr2  Fzprim::PackPaddedr   r1  )
r   r^   rF  rJ   r   
TensorTypegetrU   r_   Z	_cast_Int)r   r   lengthsr  r   r   r   _pack_padded_sequence+  s    r  c                 C   s6   | j d||dd\}}|r.| j d|g dd}||fS )Nzprim::PadPackedr   r  r  r  r  r   )r   r  r  r  padding_valuetotal_lengthr  r   r   r   _pad_packed_sequence<  s    r  c                 G   sz   t |dd}|d u rt jj}t |d}t |rl| jd|tjdgt j	d dd}| jd	|t j
| d
S | jd|dS )Nrz   re   r   r  r   r|  rd   rH   RandomNormalLikedtype_iRandomNormalshape_ir   r   r  r
  r  r/   r   rJ   rK   r   r   r   shapesre   r  r   Zshape_constr   r   r   randnI  s$    
r  c                 G   sz   t |dd}|d u rt jj}t |d}t |rl| jd|tjdgt j	d dd}| jd	|t j
| d
S | jd|dS )Nrz   re   r   r  r   r|  rd   rH   RandomUniformLiker  RandomUniformr  r  r  r   r   r   rand^  s$    
r  c                 C   s4   t |dd}|d u rt jj}| jd|t j| dS )Nrz   re   r  r  r	  r   r$   re   r   r  r  r  r   r   r   
randn_likes  s    r  c                 C   s4   t |dd}|d u rt jj}| jd|t j| dS )Nrz   re   r  r  r	  r  r   r   r   	rand_like~  s    r  c                 C   s    | j d|||d}|  d||S )Nr  )high_flow_fri  r   )r   r   rc  upperrx  r	  r  r   r   r   rrelu  s    r  c                 C   s   |d urt dd |d ur2t |s2t dd t |}|d u rPt ddS | jd|ddt j| d}| d	||}| jd
|t j| dS )N	Bernoulliz,out parameter is not supported for bernoulliz(generator is not supported for bernoulliinput dtype not accessibler  rF   r   )r  r  r  r8  rX   rZ   )r   r4   r  r   r   r]   )r   r   r	  ra   re   r  r  r   r   r   	bernoulli  s(    
r  c                 C   s   |  d|}|  d|S )Nr   r  r   )r   r   r  r   r   r   log_sigmoid  s    r  c                 C   s   |  d|S )NErfr   r   r   r   r   erf  s    r  c                 C   s   t |}|d u rt ddS |dk r.|| }|dkrR||d krR| jd||dS |dkrz||d krz| jd||d dS t | ||||S )Nr~   rP  r   r,   FlattenrY  r   )r   r   r4   r   _flatten_helper)r   r   	start_dimend_dimr~   r   r   r   flatten  s    
r  c                 C   s   t | | d|S )z/Emitted from `torch.nonzero(x, as_tuple=False)`NonZero)r   r   r   r   r   r   rM    s    rM  c                 C   s   t | t| |d|dS )Nr,   )rJ  )rV  rM  )r   r   rJ  r   r   r   nonzero_numpy  s    r  c                 C   s   |  d|}|S )NZIsNaNr   )r   r   r  r   r   r   isnan  s    r  c                 G   s~   t |dkr|d }d\}}n$|\}}}t|dg}t|d}t| |d}tj| |||d}t| || jdtdgdS )	Nr,   r   r>   rz   Fr   rG   rH   )	r?   r   r  r   r  r1  r   rJ   r  )r   rB   r   r~   r   Z	input_sumr   r   r   _any  s    

r  c              	   G   sP   |  d|d }t|dkr.|  dt| |S |  dt| ||d |d S d S )Nrh   r   r,   r   )r   r?   r  )r   rB   r   r   r   r   _all  s    r  c                 C   s   t j| ||g|g|| gdS )NrX  r  )r   r   r~   r)  lengthr   r   r   narrow  s    r  c              	   C   sj   t |r<t | || jdtdgd}| jd|dddS t |d}t |d}| jd|||dS d S )	NrG   r  rH   r  r   Fr  rz   r   r  r"   r   rJ   rK   r  r   r   r~   r   	flattenedr   r   r   argmax  s    
r  c              	   C   sj   t |r<t | || jdtdgd}| jd|dddS t |d}t |d}| jd|||dS d S )	NrG   r  rH   r  r   Fr  rz   r  r  r   r   r   argmin  s    
r  c                 C   s   |   }t|}t|r4| jd||||dS |   |krb| jd|tj|    d}| jd||t| |||dS d S )NScatterrY  rX   rZ   )r^   r_   r   r3   r/   r   r]   r  )r   r$   r~   r   srcsrc_typer   r   r   r    s    

r  c                 C   s   t |}|d u rt ddS t jt j| }t j| }t j|dd}|rf| jdt	j
||dd}nt j|}t| ||}t | ||||}t| ||S )Nscatter_addr  F)allow_nonstaticrG   rd   rH   )r   r   r4   r   r   r]   r   r  r   rJ   r  r  _scatter_helperr-   )r   r$   r~   r   r  re   r  to_addr   r   r   r  &  s     

r  c              	   C   s*   d}|  dt| || j dt|gdS )Ng9B.?rW   rG   rH   r  )r   r$   _ln2r   r   r   log2;  s    r  c                 C   s6   t |r | jdtdgdS | jdtdgdS NrG   r,   rH   r   )r   r`   r   rJ   
BoolTensorrx   r   r   r   is_floating_point@  s    
r  c                 C   sL   t |r@t |r*| jdtdgdS | jdtdgdS t| ||S r  )r   r  r   rJ   r  r-  r<   r   r   r   __is_F  s
    

r  c                 C   s   t | ||S r!   )r  r<   r   r   r   __isnot_N  s    r  c                 C   sR   | j dtddgd}|  dv r>| j d|tjd d}| j d	|||d
dS )NrG   r   r,   rH   )ByteCharr2  ShortrX   rY   rZ   OneHotr  rY  )r   rJ   r  r^   r_   r   r]   )r   r$   num_classesr  r   r   r   one_hotS  s    r  c           	   	   C   s   t |drt ddS |  }| jdtddgd}t| || jdt|gd}| jd| jd	||||d
t j	| d}| dt 
| ||d g|}t j| ||gddS )Nrz   gatherzsparse_grad == TruerG   r   r,   rH   rX   r  rY  rZ   r;   r   )r   r  r4   r^   r_   r   rJ   r  r  r]   r   r  )	r   r$   r~   r   sparse_gradre   r  depthr=   r   r   r   r  ]  s    r  c              	   C   s<  |d u r(| j d|dd}|}t| |}nb| j d|||d}| j d||dd}|  d|}| j d|| j dt|d	dd
}| j d|dd}|  d||}	|  d|	|	}
|d u rdn|}| j d|
||d}|d u rd}|dkr4| j d|tjd d}| j dtj|tjdd	}|  d||}|  d||  d||}||fS )Nr   r   r  r   r,   r   r%  rG   rH   rY  r   r8   r;   rX   r\   rZ   rd   rW   )r   rA  rJ   rK   r   r]   ru   )r   r   r~   
correctionr   r   Zt_meannum_elementsZredudced_dimsZsub_vZsqr_subZkeepdim_meanvarro   r=   r   r   r   	_var_meano  s8    
r  c                 G   s"   t | |g|R  \}}| d|S r   var_meanr   r   r   rB   r  r  r   r   r   std  s    r  c                 G   s   t | |g|R  \}}|S r!   )r  r  r   r   r   r    s    r  c                 G   s6   t |dkr t| |d |d d S t| |g|R  S d S )Nr,   r   )r?   r  )r   r   rB   r   r   r   r    s    r  c                 G   s&   t | |g|R  \}}| d||fS r   r  )r   r   rB   r  r   r   r   r   std_mean  s    r  c                 C   s   | j d|||dS )NReduceLogSumExpr   r   )r   r   r~   r   r   r   r   	logsumexp  s    r	  c           
         s  t  r jdg|R  S dd } fdd}t|dksFt|dkrt|dkrXd }n||d }t j |d	 |d
\}}}}t  |d	g}||}t  t t ||d d dg}	 j	d|	t j
| dS t|dkst|dkrt|dk rd }n||d }t j |d	 |d |d |d\}}}}t  |d	g}t  |d	g}t  |d	g}| 	d 	d|||}t  t t |d d d dg}	 	d 	d|	||}	 j	d|	t j
| dS t|dkr||d }t j |d	 |d |d\}}}}t  |d	g}t  |d	g}| 	d||} 	dt  t t ||g|dd  R  dg|}	 j	d|	t j
| dS tdtt| d d S )Nr  c                 S   s   t | d} | S )Nrz   )r   r  rd   r   r   r   _get_arange_dtype  s    z!arange.<locals>._get_arange_dtypec                    s,   t | r( jd d| t jd d} | S )NrX   r  r0  rZ   )r   r`   r   r   )range_tensorrv  r   r   _float_step_convert  s    

z#arange.<locals>._float_step_convertr   r
  r,   r   )r*  re   rX   rZ   r0  rE  r  )r)  r*  r  re   rW   r8   r)   r;   r|  )r)  r*  re   z&Unknown aten::arange signature taking z arguments.)r   r   r  r?   _arange_cast_helperr   rW  rM  r   r   r   r   rg  )
r   rB   r
  r  re   r*  r)  r  r  arange_tensorr   rv  r   r    st    	
&r  c           
      C   sT   t | |d }t| t| ||t| || jdtjdtjdd}	t| t	| ||	|S )NrG   r,   rd   rH   )
r   _arange_helperrC   r7   r   rJ   rK   rj   r-   r=   )
r   r)  r*  stepsre   r   r  r  r  r  r   r   r   linspace  s    
 r  c                 C   s   |S r!   r   rx   r   r   r   lift  s    r  c              	   C   s0   t | |d}t|}| d|t| |||S )NFrL  )Z
_cast_Boolr   r3   r   r   )r   r$   maskrN   r   r   r   masked_fill  s    
r  c                    s4  t  rjd||ddS t |r0t |}n|g}fddfdd|D }t|dkrxt j|d	|d	 d
dS dd t|D  t d	kr|S t dkrt| d	 | d	  S t 	|}|d u rt
dtdttj d d d  t }t|fddt|D jd|  fddt|D  d}jd||d}| d  } d  }t|d ddD ]@}d| |  |}	d||	}d| |  }qt|d	|}t|}
 tt d	  d d krjdtdgdg fddt|D  }jdg|R d d	i}t ||}ttd d	 d d	g tt d	 d || d  }jd||d}fd!dt d	 D |
g  fd"dt d	 |D  }jdg|R d d	i}n.jd|
g fd#dt|D R d d	i}t ||S d S )$Nr   r  )r<  c                    s`   t | s\|   dks*|   dkr\tjdk r<tdtd t 	 t
 | dg} | S )Nr  r  r*   z?Exporting masked indices are only supported after ONNX opset 9.zExporting aten::index operator with indices of type Byte. Only 1-D indices are supported. In any other case, this will produce an incorrect ONNX graph.r,   )r   r  r^   r_   r	   rq  rU   r'  r(  rW  rM  )r   rv  r   r   try_mask_to_index!  s    

z index.<locals>.try_mask_to_indexc                    s   g | ]} |qS r   r   )r   r  )r  r   r   r   1  r  zindex.<locals>.<listcomp>r,   r   F)apply_reshapec                 S   s   g | ]\}}t |s|qS r   )r   r  )r   rz   r  r   r   r   r   I  s   zUnsupported aten::index operator of advanced indexing on tensor of unknown rank, try turning on shape and type propagate during export: torch.onnx._export(..., propagate=True).z=Exporting aten::index operator of advanced indexing in opset z8 is achieved by combination of multiple ONNX operators, z2including Reshape, Transpose, Concat, and Gather. zVIf indices include negative values, the exported graph will produce incorrect results.c              
      s0   g | ](} j d  j dt|gdddqS )r%  rG   rH   r   rY  )r   rJ   r  )r   r~   )r   shape_tensorr   r   r   f  s   r  c                    s   g | ]}| vr|qS r   r   r  )adv_idx_indicesr   r   r   t  r  r  r  rY  r  r   r;   r)   rG   rH   c                    s   g | ]}| vr| qS r   r   r  r  dim_tensor_listr   r   r     s   r   r   c                    s   g | ]} | qS r   r   r  )r  r   r   r     r  c                    s   g | ]}| vr| qS r   r   r  r  r   r   r     s   c                    s   g | ]}| vr| qS r   r   r  r  r   r   r     s   )r   r   r  r  r|   r?   r  rs  r  r   r   r'  r(  rg  r	   rq  r   r  r   r   rJ   r  r"   )r   r$   r   r*  r9  Zadv_idx_countZcum_adv_index
multiplierrz   Z	adv_indexZcum_adv_index_shape_tensorZfolded_adv_idx_shape_listZfolded_adv_idx_shapeZadv_idx_permuteZfinal_shape_listfinal_shaper   )r  r  r   r  r  r   r     s    




	r   c                 C   s   d }|d u rzt |r<t | |dg}| jdtdgd}t |}|d u rZt ddS |dkrpt |d}qd	dg}n8t	|dkrt |r| jdtdgd}t |d}|rt
| |||||S t| |||||S )
Nr  rG   r   rH   r~   (Input rank must be known at export time.r,   rE   r   )r   r  r"   r   rJ   r  r   r4   r  r?   linalg_vector_normlinalg_matrix_norm)r   r$   ordr~   r   re   	ord_valueself_dimr   r   r   linalg_norm  s(    



r#  c                 C   s   |d u rt | |dg}d }|tjkrB| jd| d|||d}n|tj krj| jd| d|||d}n||dkrt ddd	d
S | jdt|gd}t j| | d| d||||d}| d|| d| jdtdgd|}|S )Nr  r  r  r   r  r   r  r*   r+   zord=0 not supportedrG   rH   rC  rW   r,   )	r   r"   r  infr   r1   rJ   r5  r  )r   r$   r   r~   r   re   resultZord_opr   r   r   r    s*    
r  c              	   C   s  t |d}|dkr"t| |||S |dkr6t ddS t |d}|d u rXt| |||S |dksh|dkrtt dd	S t |}|d u rt dd
S |d dk r|d  |7  < |d dk r|d  |7  < |tjks|tj k r|d |d  |d< |d< |d |d kr$|s$|d  d8  < t j| | d||d g|d}|dkrzt	| || jdt
|d gd|d\}	}
n*t| || jdt
|d gd|d\}	}
|	S d S )NrP   fronuczlinalg.matrix_normzord==nucrE   r   r  zord==2r  r   r,   r  r   rG   rH   )r  r   )r   r  frobenius_normr4   r   r  r$  r  r   r  rJ   r  r  )r   r$   r   r~   r   re   r!  r"  r   r%  r*  r   r   r   r    sP    


r  r  c                 C   s   t | |||S r!   )r  )r   r   r'   r~   r   r   r   linalg_cross%  s    r)  c                 C   s,   |  d||}tj| |||d}|  d|S )Nr;   r   r   )r   r   r  )r   r$   r~   r   sqrZsumsqrr   r   r   r(  *  s    r(  c                 C   sX   |d urt |st dd |s6|dkr6t dd t| |}| jd|t jd |dS )NMultinomialz*generator is not supported for multinomialr,   zGreplacement=False when num_samples > 1 is not supported for multinomialrY   )r  Zsample_size_i)r   r  r4   r  r   r]   )r   r   num_samplesreplacementr	  	log_inputr   r   r   multinomial1  s     
r/  c           
      C   s`   |   }t| ||}t| || jd|tj| d}t| || jd|tj| d}	t| ||	S r  )r^   r_   r   r=   r   r   r]   r-   )
r   r$   batch1batch2r   r6   re   Z	batch_mulZmul_aZmul_br   r   r   baddbmmF  s    r2  )indexingc                    s8  |d u rd}n|dvr$t d| |dkrF|d |d  |d< |d<  fddt|D } fd	d|D } jd
g|R ddi}g }t|D ]l\}} jdtjdtjddgt| }	|| |	|< t	 | jd
g|	R ddi}
|
 d|
| q|dkr&|d |d  |d< |d<  jdg|R  S )Nij>   r4  xyzUnsupported indexing: r5  r,   r   c                    s,   g | ]$}t  | jd tdgdqS )rG   r  rH   )r   r"   r   rJ   r  r   rv  r   r   r   \  s   zmeshgrid.<locals>.<listcomp>c                    s   g | ]}  d |qS )r   r   r   rv  r   r   r   b  r  r   r   rG   rd   rH   r  prim::ListConstruct)
ValueErrorr   r|   r   rs  rJ   r   rj   r?   r    r   )r   r}   r3  r   Ztensors_shape	out_shapera   rz   r   r  Z
t_reshapedr   rv  r   meshgridT  s,    
 
r9  c                 C   s(   t | ||}| d||}| d||S )Nr;   r8   )rS   r   )r   r   r'   rC   quor   r   r   	remainderq  s    r;  )r$   approximatec                 C   s&  |dkrt dt j }d}tj|tjd}tj|tjd}tjdtjd}tjdtjd}t| |t| ||}	t| |t| |t| ||	}
t| |t| |t| || d|
S d}| d	| d
|tj|tjd}t| || jdtjdtjdd}t| t| ||| jdtjdtjddS d S )Nr   r   gHm?rd   rF   g      ?r   g;f?r  rW   rG   r,   rH   )	r  r   pirJ   rK   rv   r=   r-   r   )r   r$   r<  ZkBetaZkKappar   kapparo   halfZ	self_cubeinnerZ_sqrt2r  Zerf_plusoner   r   r   geluw  s(    $"
rA  c              
   C   s  t  r | jd||||||dS t |d}|d urD|| dksDJ t |}|d u rbt ddS d|dg}	t | || jdt	|	d}
| jdt
d	g| d
|   d d}| jdt
dg| d
|   d d}| jd|
|||d}t | || d|}|d u s,|  r\t
d	gd
|   d }| jd|d}|d u st|  rt
dgd
|   d }| jd|d}ttd|d }t| t| |t | ||t | ||S )N
group_norm)Znum_groups_ir}  Zcudnn_enabled_ir,   r   zunknown input rankr  rG   rH   rF   r  r  r   r  r  r   )r   r   r  rH  r   r4   r"   r   rJ   r  rK   r^   r_   r   
mustBeNoner   r  r-   r=   r   )r   r   
num_groupsr)  rV  rz  r\  r  
input_rankr   r  r  r  Znorm_reshapedr  r  r  r   r   r   r   rB    sh    



rB  c                 C   s   t |}|d urttt|}|d urH|dk r6||7 }|dkrH|| t| |d|d}| d||}| d||S t  r| jd|||dS t	dd S )	Nr  r   r,   rW   r;   _weight_normr   zDUnsupported: ONNX export of _weight_norm for tensor of unknown rank.)
r   r   r   r  remover  r   r   r  rU   )r   r  weight_gr~   r9  r   norm_vrC   r   r   r   rF    s     

rF  c                 C   s   |  d|}|  d|S )zFImplement the dim functionality available for a pytorch tensor in ONNXr   Sizer   r#   r   r   r   r~     s    r~   c                 C   s    t | || jdtdgd|S r  )r\  r   rJ   rK   )r   r$   rz   r   r   r   
__getitem_  s    rK  c                 C   s   |S r!   r   rx   r   r   r   rJ    s    rJ  c              
   C   sD   t | || jdtjdgtjdd}t| |d|}t| ||}|S )NrG   r  rd   rH   r   )r   r"   r   rJ   rK   rj   r  r(   )r   r$   r   Zself_flattenedra   r   r   r   take  s    rL  c                 C   s&   t | ||}t| |}t| ||}|S r!   )r7   r  r=   )r   r   targetdiff_exp_r  r   r   r   _kl_div_log_target_impl  s    
rP  c           	      C   sZ   t | |}t| ||}t| ||}t| |}t| || jdtdd}t| |||}|S r  )	r  r7   r=   r  r1  r   rJ   rK   r  )	r   r   rM  log_rN  
output_poszeros_Zmask_r  r   r   r   _kl_div_non_log_target_impl  s    

rT  c                 C   sh   |rt | ||}nt| ||}|dkr*|S |dkrB| jd|ddS |dkrZtj| |ddS tdS d S )Nr   r,   r   r  r   z4kl_div with reduction other than none, mean, or sum.)rP  rT  r   r   r  r  )r   r   rM  	reduction
log_targetr  r   r   r   kl_div  s    rW  c                 C   s  t |d}t|}t | || jdtjdgtjdd}t |stjdgtj	d}t
t||D ]6\}\}	}
dg| }d||< |t|	||
  }qd|r|| }| d|| jd|dS d }t
|D ]\}}
dg| }d||< t| || jdtdgd| jdt|d}	t | t| |	d	d d d | jdt|d}| d
|| jdt|
gd}|d u rr|}q| d||}q|r| d|| dt|g}| d||S d S )Nr   rG   r  rd   rH   r   r,   r%  r0  r;   r)   )r   r  r?   r"   r   rJ   rK   rj   r/   longrs  r  r  r@  r\  )r   r$   r  stridesoffsetr9  Zself_1dindrz   r  r  Zr_sizeZtmp_indr   r   r   
as_strided*  sL    



r\  c              	   C   s   |  d||  d||S )Nr)   r;   r   )r   r   r)  r  r   r   r   __derive_indexW  s    r]  c                 C   s8   |  d||}|  dt| ||}| j d|tjd dS )Nr8   r  rX   rY   rZ   )r   r@   r   r]   )r   lor  r  r7   rC   r   r   r   __range_lengthc  s    r_  c                 C   s   t |}t| |}|dkrp|  sp| jdtjdtjdd}| jdtjdtjdd}t	| |||||}n$t
| ||}|  st| ||}|S )Nr   rG   r,   rd   rH   )r   r   r   r   rC  r   rJ   rK   rj   r   r   r-   )r   r   r)  rV  r9  r6   r   r  r   r   r   r  i  s    

c           
   	   C   s   |d u r,t  }|r|js t j}tj|}t| |dd d d }| jd|tj	d d}	t
| | jdt jtjt jdd|	}	|du rt| || jdt jd	t jdd}t| |	|}	| jdt| t| |	tj| d}	|	S )
Nr0  rX   r\   rZ   rG   rd   rH   Fr,   )rJ   rt   r  ru   r   r   r   r  r   r]   r=   rK   r  r=  r7   r;  rC   r]  r   r   )
r   window_lengthperiodicre   r   r  r  r  Zn_arrayr  r   r   r   hann_windowx  s*    
rb  c                 C   s   t | ||S r!   r   )r   r$   vecr   r   r   mv  s    re  c                 C   s   t | ||S r!   rc  r<   r   r   r   dot  s    rf  c                 C   s>   |   }|d u rtjj}ntjtj| }t| |||S r!   )	r^   r_   r   r  r
  r   r   r]   r!  )r   r$   rN   re   r   r   r   fill  s    
rg  c                    s  t d |r.tt|dkr.tddS t d  d u rJtdt|}t|}|d u sn|d u rvtd||kr|| }t	|D ]}	t
| |t|g}qt| }
t| }|
d ur|d ur|
|krtdtt	|}d	d
 t	|D } fdd
t	|D }tj| ||||d}t| ||}t	 D ]}	t
| |dg}q>t	|  d D ]}	t
| |t|g}qft| | t| |||S )NzyWarning: ONNX export does not support duplicated values in 'index' field, this will cause the ONNX model to be incorrect.r,   	index_addr.   rz   zXONNX export does NOT support exporting 'index_add_()' function with unknown 'dim' value.z~ONNX export does NOT support exporting 'index_add_()' function while the rank of self tensor or tensor to be added is unknown.zoONNX export does NOT support exporting 'index_add_()' function with duplicated values in 'index' parameter yet.c                 S   s   g | ]}d qS r  r   r  r   r   r   r     r  zindex_add.<locals>.<listcomp>c                    s   g | ]}| krt jnd qS r  )sysmaxsizer  r~   r   r   r     r  rX  r   )r'  r(  r   r2   r3   r4   r  r   r   r  r   rH  r   rZ  r  r  )r   r$   r~   r   r'   r6   Zself_dim_rankZother_dim_rankdeltarz   Zother_dim_sizeZself_dim_sizeZnew_shape_axesZnew_shape_startsZnew_shape_endsr  r   rk  r   rh    sV    


rh  c                 C   s   t |t |ksJ |}tt |D ]}g }tj| ||| g||  gtjgd}|| tj| ||| gdg||  gd}|| | jdg|R d|| i}q$|S )NrX  r   r   r   )r?   r  r   rZ  ri  rj  r   r   )r   r$   shiftsr"  r%  rz   r  r   r   r   r   roll  s    

rn  c                 C   sp   t ||}t| |dg|g}t| |dg|g}t| |dg|g}t| |dg|g}t| t| ||t| ||S )Nr   r,   )r   _get_dim_for_crossrn  r7   r=   )r   r   r'   r~   Zroll_x_1Zroll_y_1Zroll_x_2Zroll_y_2r   r   r   r    s    r  r~  #use_mm_for_euclid_dist_if_necessaryc                 C   sF   t |}t | ||d g}t | ||d g}t| |||dddS )Nr,   r   gư>F)rz  r   )r   r   r   r  )r   r  r  r  compute_moder9  Zbroadcasted_x1Zbroadcasted_x2r   r   r   cdist  s    
rr  c                    sT   t |}t |d |D ]}t |q fdd|D } jdg|R  S )Nr   c                    s   g | ]}t  |qS r   )r  r   r   Zt_with_final_shaper   r   r   2  r  z%broadcast_tensors.<locals>.<listcomp>r6  )r   r|   r  r-   r   )r   r$   all_tensorsr   t_listr   rs  r   broadcast_tensors)  s    
rv  c                   @   s  e Zd ZdZedd Zedd Zedd Zedd	 Zed'ddZ	edd Z
edd Zeeeej  dddZedd Zedd Zedd Zedd Zedd Zeejjddd Zeejjdd!d"Zeejjdd#d$Zeejjdd%d&Zd
S )(Primprimc                 C   s\   t ||}|d u r t ddS |g||  }|| }|rD|| | jd|||t|dS )Nzprim::ConstantSplitrD  rE  rF  )r   rH  r4   r   r   r?   )r   r$   rK  r~   r  rL  rM  r   r   r   ConstantSplit9  s    
zPrim.ConstantSplitc                 C   s@   t ||}|d u r t ddS || d | }t| |||S )Nzprim::ConstantChunkrD  r,   )r   rH  r4   rw  ry  )r   r$   rI  r~   rh  rK  r   r   r   ConstantChunkJ  s    zPrim.ConstantChunkc                 C   s   |  d|S r   r   rx   r   r   r   r   T  s    z
Prim.shapec                 C   s   t | d||ddS )Nr  r  ry  rz  r<   r   r   r   r  X  s    zPrim.maxNc                 C   sB   |s6t |r,t| || jdtdgd}t| |S t| ||S r  )r   r  r   r   rJ   rK   r  r<   r   r   r   r  \  s
    

zPrim.minc                 C   s   |S r!   r   rx   r   r   r   r  d  s    z	Prim.datac                 O   s   d S r!   r   r   ru  r   r   r   r   ListConstructh  s    zPrim.ListConstruct)returnc                 O   s2   t |dkr.|d   dkr.t|d S d S )Nr,   r   r6  )r?   r   r   r   r|   r{  r   r   r   
ListUnpackl  s     zPrim.ListUnpackc                 O   s   d S r!   r   r{  r   r   r   TupleConstructu  s    zPrim.TupleConstructc                 O   s   d S r!   r   r{  r   r   r   Uninitializedy  s    zPrim.Uninitializedc                 C   s   |S r!   r   rx   r   r   r   unchecked_cast  s    zPrim.unchecked_castc                 C   s<   t |}|d u rd}t jt j| }| jdt|dS )Nr\   rG   rH   )r   r   r   r   r]   r   rJ   rK   r   r   r   r   re     s    
z
Prim.dtypec                 C   s$   t |d}|dkr t ddS |S )Nrz   r,   zprim::tolistzdim_val > 1)r   r  r4   )r   r   dim_valZelem_ty_valr~   r   r   r   tolist  s    zPrim.tolistctxc                 O   s*   | j }|   dkrd S tddS )Nr+  rD  z#output type is not `DeviceObjType`.)cur_noder  r^   r   r   r4   )r  r   ru  r   r   r   r   r   r    s    zPrim.devicec                 O   s(  | j }| j}| j}tj}tj}|jdg|R d| i}	| dkrR|	d  n|	 }
|	 D ]}|

 }t| D ]l\}}|dkr|t|k r|||   |dkrz|d t|k rzt| tjsz|||d    qztj||||d qbtj|
|}	tjr$tj|
|| |	S )NLoopr1  r,   r   F)r  envparams_dictr	   r&  rq  r   outputsSizer   blocksaddBlockrs  ru  r?   r   r^   r   r   r   rJ   _jit_pass_onnx_block%_jit_pass_fixup_onnx_controlflow_nodeonnx_shape_inference(_jit_pass_onnx_node_shape_type_inference)r  r   ru  attrsr   r  r  r&  opset_versionnew_op_outputsnew_noder$  	new_blockrz   Zb_inr   r   r   r    s>    
z	Prim.Loopc                 O   s  | j }| j}| j}| j}tj}tj}	|d   dk}
|
r|d  d 	 }t
|trdt|nt|}|rtdnd}t| | }tj||||d}t| }t| }g }tt|D ]8}|| |vrtd|| |||  }|| q|S |jdg|R d| i}| dkr8|d  n| }| D ]"}| }tj||||d	 qHtj||	}tjrtj|||	 |S d S )
Nr   r   rN   r,   Tz+The sub block ATen output {} is not in env.Ifr1  F)r  
onnx_blockr  r  r	   r&  rq  r   r   r  r   r   allr,  r  rJ   r   r  r1  r  r?   rU   r   r   r   r  r  r  r  r  )r  r   ru  r  r   blockr  r  r&  r  Z	static_ifZ
input_flagr  	block_idxZ	current_bZif_output_listZcurrent_b_listZfinal_b_listr  Zonnx_br  r  r$  r  r   r   r   r    sn    
zPrim.Ifc                 O   s   | j }| rd S t|  tjr*d S |ddkrJ|jd|d dS |ddkrj|jd|d dS |  	tj
 s|  	tj
 r|jdt|d dS td|dd S )NrN   r   rG   rH   rP   )Zvalue_sz9Unsupported prim::Constant kind: `{}`. Send a bug report.)r  rC  r   r  r^   r   r+  kindOfr   rF  rG  rH  ofFloatsrJ   rK   rU   r   )r  r   ru  r  r   r   r   r   rG   .  s(    zPrim.Constant)N)r   
__module____qualname__domainstaticmethodry  rz  r   r  r  r  r|  r   r   r   r  r~  r  r  r  re   r  rJ   onnxSymbolicContextr  r  r  rG   r   r   r   r   rw  6  sF   

	








	
-Xrw  c                   @   s&   e Zd ZdZeejjdddZdS )Onnxr  r  c                 O   s"   | j }| j}| j}tj|||S r!   )r  r  r  rJ   r   '_jit_onnx_convert_pattern_from_subblock)r  r   ru  r  r   r  r  r   r   r   PlaceholderO  s    zOnnx.PlaceholderN)	r   r  r  r  r  rJ   r  r  r  r   r   r   r   r  I  s   r  )N)N)N)rF   )T)T)N)N)N)N)N)N)r   N)N)F)N)N)NNN)N)N)FF)NN)NN)FN)NNNFN)F)NNF)NN)F)NNNFN)F)F)NNNFN)F)F)NNNFN)F)N)N)NN)NN)NNFN)NNFN)NN)N)F)r  )NF)FN)N)r   )N)TNNNNF)N)N)r~  rp  (`  __doc__r   r  ri  r'  typingr   r   r   r   rJ   Ztorch._C._onnxr   _onnxr  Ztorch.nn.modules.utils
torch.onnxr   r   torch.onnx._globalsr	   r   r   r    r   r(   r-   r7   r9   r=   rC   r   rO   rA   rT   rS   rq   rr   r@   ry   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   r   r   r   r   r   r  r
  r   r  r  r.  r/  r  r:  r?  r@  rA  rB  rN  rR  rP  rU  rV  r\  r]  rd  rj  rk  rm  rv  r{  r~  r  rQ   r  r  r  r  r  r  r  r  nnmodulesutils_singler  _pairr  _tripler  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  reflection_pad1dreflection_pad2dreflection_pad3dreplication_pad1dreplication_pad2dreplication_pad3dr  r  r  r  r	  r  r  r  r  r  r"  r%  r)  r*  r-  r.  r1  r0  r7  r6  r9  r:  r<  r>  r?  r@  rA  rB  rI  rK  r  rR  r`  rd  re  rf  ri  rj  rk  rl  r|  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  dropout_feature_dropout_alpha_dropout_feature_alpha_dropout_r  r  r  r  r]   itemsr  rD   r   r   partial_cast_func_templater$  r  r  r  r  rK   r  r  r  r  r   r  r  r  r!  r"  r$  r+  r.  r1  r/  r3  r8  r9  r:  r  r@  rA  rC  r  r  rO  rW  ra  r  r  r  r  r  r  grurnn_tanhrnn_relur  r  r  r  r  r  r  r  r  r  r  r  r  r  rM  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  r  r  r  r  r   r#  r  r  r)  r(  r/  r2  rg  r9  r;  r  rA  rB  rF  r~   rK  rJ  rL  rP  rT  rW  r\  r]  r_  r  rb  re  rf  rg  rh  rn  r  rr  rv  rw  r  r   r   r   r   <module>   s  "
 




4

$


	

!



>1


	

	<(:


//


(					




@6!Z










    
G


F]
D
L  aB



"
Q
 "4
"
&A	
,     $
B  