a
    NSic]K                     @   s   d dl Z d dlZddlmZ eejdsXedejjd< edejjd< edejjd< d dlmZ d d	lm	Z	 d d
lm
Z
 dd Zdd ZG dd dejjZG dd deZdd ZdS )    N   )_dummy_type_CudaStreamBase
_CUDAGraph_graph_pool_handle_cuda_isCurrentStreamCapturing)r   r   r   c                   C   s   t  S )z
    Returns True if CUDA graph capture is underway on the current CUDA stream, False otherwise.

    If a CUDA context does not exist on the current device, returns False without initializing the context.
    r	    r
   r
   M/var/www/html/django/DPS/env/lib/python3.9/site-packages/torch/cuda/graphs.pyis_current_stream_capturing   s    r   c                   C   s   t  S )z
    Returns an opaque token representing the id of a graph memory pool.
    See :ref:`Graph memory management<graph-memory-management>`.

    .. warning::
        This API is in beta and may change in future releases.
    r   r
   r
   r
   r   graph_pool_handle   s    r   c                       sj   e Zd ZdZ fddZ fddZd fdd	Z fd	d
Z fddZ fddZ	 fddZ
  ZS )	CUDAGraphzw
    Wrapper around a CUDA graph.

    .. warning::
        This API is in beta and may change in future releases.
    c                    s   t t| | S N)superr   __new__)cls	__class__r
   r   r   .   s    zCUDAGraph.__new__c                    s   t t|   d S r   )r   r   __init__selfr   r
   r   r   1   s    zCUDAGraph.__init__Nc                    s,   |du rt t|   nt t| | dS )aX  
        Begins capturing CUDA work on the current stream.

        Typically, you shouldn't call ``capture_begin`` yourself.
        Use :class:`~torch.cuda.graph` or :func:`~torch.cuda.make_graphed_callables`,
        which call ``capture_begin`` internally.

        Arguments:
            pool (optional): Token (returned by :func:`~torch.cuda.graph_pool_handle` or
                :meth:`other_Graph_instance.pool()<torch.cuda.CUDAGraph.pool>`) that hints this graph may share memory
                with the indicated pool.  See :ref:`Graph memory management<graph-memory-management>`.
        N)r   r   capture_begin)r   poolr   r
   r   r   4   s    zCUDAGraph.capture_beginc                    s   t t|   dS )aP  
        Ends CUDA graph capture on the current stream.
        After ``capture_end``, ``replay`` may be called on this instance.

        Typically, you shouldn't call ``capture_end`` yourself.
        Use :class:`~torch.cuda.graph` or :func:`~torch.cuda.make_graphed_callables`,
        which call ``capture_end`` internally.
        N)r   r   capture_endr   r   r
   r   r   H   s    	zCUDAGraph.capture_endc                    s   t t|   dS )z?
        Replays the CUDA work captured by this graph.
        N)r   r   replayr   r   r
   r   r   S   s    zCUDAGraph.replayc                    s   t t|   dS )zD
        Deletes the graph currently held by this instance.
        N)r   r   resetr   r   r
   r   r   Y   s    zCUDAGraph.resetc                    s   t t|  S )z
        Returns an opaque token representing the id of this graph's memory pool.
        This id can optionally be passed to another graph's ``capture_begin``,
        which hints the other graph may share the same memory pool.
        )r   r   r   r   r   r
   r   r   _   s    zCUDAGraph.pool)N)__name__
__module____qualname____doc__r   r   r   r   r   r   r   __classcell__r
   r
   r   r   r   '   s   r   c                   @   s.   e Zd ZdZdZd	ddZdd Zdd ZdS )
grapha  
    Context-manager that captures CUDA work into a :class:`torch.cuda.CUDAGraph`
    object for later replay.

    See :ref:`CUDA Graphs <cuda-graph-semantics>` for a general introduction,
    detailed use, and constraints.

    Arguments:
        cuda_graph (torch.cuda.CUDAGraph): Graph object used for capture.
        pool (optional): Opaque token (returned by a call to :func:`~torch.cuda.graph_pool_handle()` or
            :meth:`other_Graph_instance.pool()<torch.cuda.CUDAGraph.pool>`) hinting this graph's capture
            may share memory from the specified pool. See :ref:`Graph memory management<graph-memory-management>`.
        stream (torch.cuda.Stream, optional): If supplied, will be set as the current stream in the context.
            If not supplied, ``graph`` sets its own internal side stream as the current stream in the context.

    .. note::
        For effective memory sharing, if you pass a ``pool`` used by a previous capture and the previous capture
        used an explicit ``stream`` argument, you should pass the same ``stream`` argument to this capture.

    .. warning::
        This API is in beta and may change in future releases.
    Nc                 C   sl   | j jd u rtj | j _|d u r&dn|f| _|d ur:|n| j j| _| jd usRJ tj| j| _|| _	d S )Nr
   )
r   default_capture_streamtorchcudaStreamr   Zcapture_streamstream
stream_ctx
cuda_graph)r   r)   r   r'   r
   r
   r   r      s    zgraph.__init__c                 C   s8   t j  t  t j  | j  | jj	| j
  d S r   )r$   r%   synchronizegccollectZempty_cacher(   	__enter__r)   r   r   r   r
   r
   r   r-      s
    


zgraph.__enter__c                 C   s   | j   | j||| d S r   )r)   r   r(   __exit__)r   exc_type	exc_value	tracebackr
   r
   r   r.      s    
zgraph.__exit__)NN)r   r   r   r    r#   r   r-   r.   r
   r
   r
   r   r"   h   s     
r"   c                     sL  d}t | tsd}| f} ft| D ]\}}t |tjjrt|jdkrht|jdkrht|j	dkspJ dt
dd | D sJ dt
dd |D s(J d	q(d
d D }dd | D   fddtt| D }dd tt| D }dd tt| D }t }	tj  tjtj  t| |D ]r\}
}}tdD ]X}|
| }t |tjrp|fn|}tjj|tdd |D tdd |D ddd}qP~~q>W d   n1 s0    Y  tj  g }g }t| |D ]z\}
}}tjj||	d |
| }W d   n1 s*0    Y  t |tjrT|d |f}n
|d || qg }g }tt|t|t|t D ]\}}}}t
dd |D sJ dtdd |D }tjj||	d4 tjj|tdd |D |ddd}W d   n1 s0    Y  g }d}|D ]0}|jrR|||  |d7 }n
|d q.t|}|| || qtt|}tt|}dd }g }t| D ]\}}
||| ||  | || || || || || || 	}t |
tjjr(dd }||
|
j||
j|
_||
 n
|| q|rD|d S t|S )a  
    Accepts callables (functions or :class:`nn.Module<torch.nn.Module>`\ s)
    and returns graphed versions.

    Each graphed callable's forward pass runs its source callable's
    forward CUDA work as a CUDA graph inside a single autograd node.

    The graphed callable's forward pass also appends
    a backward node to the autograd graph. During backward, this node runs the
    callable's backward work as a CUDA graph.

    Therefore, each graphed callable should be a drop-in replacement for its source callable
    in an autograd-enabled training loop.

    See :ref:`Partial-network capture<partial-network-capture>` for detailed use and constraints.

    If you pass a tuple of several callables, their captures will use the same memory pool.
    See :ref:`Graph memory management<graph-memory-management>` for when this is appropriate.

    Arguments:
        callables (torch.nn.Module or Python function, or tuple of these): Callable or callables to graph.
            See :ref:`Graph memory management<graph-memory-management>` for when passing a tuple of callables
            is appropriate.  If you pass a tuple of callables, their order in the tuple must be the same order
            they'll run in the live workload.
        sample_args (tuple of Tensors, or tuple of tuples of Tensors): Samples args for each callable.
            If a single callable was passed, ``sample_args`` must be a single tuple of argument Tensors.
            If a tuple of callables was passed, ``sample_args`` must be tuple of tuples of argument Tensors.

    .. note::
        The ``requires_grad`` state of each Tensor in ``sample_args`` must match the state
        that's expected for the corresponding real input in the training loop.

    .. warning::
        This API is in beta and may change in future releases.

    .. warning::
        ``sample_args`` for each callable must be a tuple of Tensors. Other types and keyword args
        are not allowed.

    .. warning::
        Returned callables do not support higher order differentiation (e.g., double backward).

    .. warning::
        In any :class:`~torch.nn.Module` passed to :func:`~make_graphed_callables`, only parameters
        may be trainable. Buffers must have ``requires_grad=False``.

    .. warning::
        After you pass a :class:`torch.nn.Module` through :func:`~make_graphed_callables`,
        you may not add or remove any of that Module's parameters or buffers.

    .. warning::
        :class:`torch.nn.Module`\s passed to :func:`~torch.cuda.make_graphed_callables` must not have module hooks
        registered on them at the time they are passed. However, registering hooks on modules *after* passing them
        through :func:`~torch.cuda.make_graphed_callables` is allowed.

    .. warning::
        When running a graphed callable, you must pass its arguments in the same order and format
        they appeared in that callable's ``sample_args``.

    .. warning::
        All Tensor outputs of graphed callables must require grad.
    FTr   zModules must not have hooks registered at the time they are passed. However, registering hooks on modules after passing them through make_graphed_callables is allowed.c                 s   s   | ]}|j d u V  qdS )FNrequires_grad.0br
   r
   r   	<genexpr>       z)make_graphed_callables.<locals>.<genexpr>zIn any :class:`~torch.nn.Module` passed to :func:`~make_graphed_callables`, only parameters may be trainable. All buffers must have ``requires_grad=False``.c                 s   s   | ]}t |tjV  qd S r   )
isinstancer$   Tensor)r5   argr
   r
   r   r7      r8   zxIn the beta API, sample_args for each callable must be a tuple of Tensors. Other types and keyword args are not allowed.c                 S   s   g | ]}t |qS r
   )len)r5   argsr
   r
   r   
<listcomp>   r8   z*make_graphed_callables.<locals>.<listcomp>c                 S   s*   g | ]"}t |tjjr"t| nd qS )r
   )r9   r$   nnModuletuple
parameters)r5   cr
   r
   r   r>      s   c                    s   g | ]}|  |  qS r
   r
   r5   iZper_callable_module_paramssample_argsr
   r   r>      s   c                 S   s   g | ]}t j qS r
   r$   r%   r   r5   _r
   r
   r   r>      r8   c                 S   s   g | ]}t j qS r
   rH   rI   r
   r
   r   r>      r8      c                 s   s   | ]}|j r|V  qd S r   r2   rD   r
   r
   r   r7     r8   c                 s   s   | ]}t |V  qd S r   r$   
empty_liker5   or
   r
   r   r7     r8   )outputsinputsgrad_outputsZonly_inputsZallow_unusedN)r   c                 s   s   | ]}|j V  qd S r   r2   rN   r
   r
   r   r7   6  r8   z/Outputs of graphed callables must require grad.c                 s   s   | ]}t |V  qd S r   rL   rN   r
   r
   r   r7   7  r8   c                 s   s   | ]}|j r|V  qd S r   r2   rD   r
   r
   r   r7   ;  r8   r   c	           
         s8   G 	fdddt jj  fdd}	|	S )Nc                       s@   e Zd ZefddZeejjj fddZ	dS )zOmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.Graphedc                    s`   t D ].}|  ||  kr| ||  q   ttsNJ tdd D S )Nc                 s   s   | ]}|  V  qd S r   detachrN   r
   r
   r   r7   g  r8   zjmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.Graphed.forward.<locals>.<genexpr>)rangedata_ptrcopy_r   r9   rA   )ctxrQ   rE   )	fwd_graphlen_user_argsstatic_input_surfacestatic_outputsr
   r   forward_  s    zWmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.Graphed.forwardc                    sl   t |D ]8\}}|d u r(|d u sBJ q
| | kr
|| q
   ttsZJ tdd D S )Nc                 s   s"   | ]}|d ur|  n|V  qd S r   rS   r4   r
   r
   r   r7   x  r8   zkmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.Graphed.backward.<locals>.<genexpr>)ziprV   rW   r   r9   rA   )rX   Zgradsggrad)	bwd_graphstatic_grad_inputsstatic_grad_outputsr
   r   backwardi  s    zXmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.Graphed.backwardN)
r   r   r   staticmethodr]   r$   autogradfunctionZonce_differentiablerd   r
   )ra   rY   rZ   rb   rc   r[   r\   r
   r   Graphed^  s
   	rh   c                     s    j |   }r|d S |S )Nr   )apply)	user_argsout)rh   module_paramsoutput_was_tensorr
   r   functionalizedz  s    zVmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.functionalized)r$   rf   Function)
rY   ra   rl   rZ   rm   r[   r\   rc   rb   rn   r
   )
rh   ra   rY   rZ   rl   rm   rb   rc   r[   r\   r   make_graphed_autograd_functionU  s    	$z>make_graphed_callables.<locals>.make_graphed_autograd_functionc                    s    fdd}|S )Nc                     s    j kr|  S |  S d S r   )training)rj   funcgraph_training_stategraphedorig_fwdr
   r   new_fwd  s    
zEmake_graphed_callables.<locals>.make_graphed_forward.<locals>.new_fwdr
   )rs   rt   ru   rv   rw   r
   rr   r   make_graphed_forward  s    z4make_graphed_callables.<locals>.make_graphed_forward)r9   rA   r^   r$   r?   r@   r<   _backward_hooksZ_forward_hooksZ_forward_pre_hooksallbuffersrU   r   r%   r*   r'   r&   r:   rf   r`   r"   appendreversedr3   list	enumeraterq   r]   ) Z	callablesrG   Zjust_one_callablerC   r=   Zper_callable_len_user_argsZ"per_callable_static_input_surfacesZ
fwd_graphsZ
bwd_graphsZmempoolrs   r[   rJ   rP   grad_inputsZper_callable_static_outputsZper_callable_output_was_tensorrY   Z per_callable_static_grad_outputsZper_callable_static_grad_inputsr\   ra   rl   rc   rb   Zgrad_idxr;   rp   retrE   ru   rx   r
   rF   r   make_graphed_callables   s    ?
,


(
(

&	

/
	r   )r+   r$   _utilsr   hasattr_C__dict__torch._Cr   r   r   r   r   r   objectr"   r   r
   r
   r
   r   <module>   s   	A<