a
    lc                     @   sX  d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z	 ddl
mZ ddlmZmZ ddlmZ eeZd	Zd
ZejZejZejZejZejZdd Zdd Zdd Zdd Zdd Zdd Z dd Z!dd Z"dd Z#dd Z$d4d d!Z%d"d# Z&d$d% Z'd&d' Z(d(d) Z)G d*d+ d+e*Z+d,d- Z,G d.d/ d/Z-G d0d1 d1Z.G d2d3 d3Z/dS )5aa  
The diff parser is trying to be a faster version of the normal parser by trying
to reuse the nodes of a previous pass over the same file. This is also called
incremental parsing in parser literature. The difference is mostly that with
incremental parsing you get a range that needs to be reparsed. Here we
calculate that range ourselves by using difflib. After that it's essentially
incremental parsing.

The biggest issue of this approach is that we reuse nodes in a mutable way. The
intial design and idea is quite problematic for this parser, but it is also
pretty fast. Measurements showed that just copying nodes in Python is simply
quite a bit slower (especially for big files >3 kLOC). Therefore we did not
want to get rid of the mutable nodes, since this is usually not an issue.

This is by far the hardest software I ever wrote, exactly because the initial
design is crappy. When you have to account for a lot of mutable state, it
creates a ton of issues that you would otherwise not have. This file took
probably 3-6 months to write, which is insane for a parser.

There is a fuzzer in that helps test this whole thing. Please use it if you
make changes here. If you run the fuzzer like::

    test/fuzz_diff_parser.py random -n 100000

you can be pretty sure that everything is still fine. I sometimes run the
fuzzer up to 24h to make sure everything is still ok.
    N)
namedtuple)split_lines)Parser)	EndMarker)PythonTokenBOM_UTF8_STRING)PythonTokenTypesF)INDENTERROR_DEDENTDEDENTc                 C   s   | j dko| jtv S )N
error_leaf)type
token_type_INDENTATION_TOKENSnode r   M/var/www/html/django/DPS/env/lib/python3.9/site-packages/parso/python/diff.py_is_indentation_error_leaf3   s    r   c                 C   s   | rt | r|  } q | S N)r   get_previous_leafleafr   r   r   !_get_previous_leaf_if_indentation7   s    
r   c                 C   s   | rt | r|  } q | S r   )r   get_next_leafr   r   r   r   _get_next_leaf_if_indentation=   s    
r   c                 C   s   t | jd S N   )_get_indentationchildren	tree_noder   r   r   _get_suite_indentationC   s    r"   c                 C   s
   | j d S r   )	start_posr    r   r   r   r   G   s    r   c           	      C   sh  z| j }W n* ty6   | jdkrJ| jtv rJ| jr:J | jrDJ Y dS t|  }|du rj| j}d}n*|j	| j
ksJ || f|j| j }|j
}d|v sd|v rt|}|d t| d }|t|d f}nF|d |d t| f}|tr|  dkr|d |d d f}| j
|ks2J | j
|fY n.0 |D ]&}|j| ksXJ | |ft| q<dS )	z~
    Checks if the parent/children relationship is correct.

    This is a check that only runs during debugging/testing.
    r   N)r   r   
r   r   )r   AttributeErrorr   r   r   valueprefixr   r   end_posr#   r   len
startswithr   get_start_pos_of_prefixparent_assert_valid_graph)	r   r   Zprevious_leafcontentZprevious_start_posZsplittedlineactualchildr   r   r   r/   K   s:    


 r/   c                 C   s  z
| j }W n ty   t|dr.J | |f| j|jksFJ | |f| j|jks^J | |f| j|jksvJ | |f| j|jksJ | |fY d S 0 z
|j }W n" ty   dsJ | |fY n0 t||D ]\}}t|| qt	|t	|ksJ dt
| d t
| d S )Nr   Fr$   )r   r'   hasattrr(   r   r)   r#   zip_assert_nodes_are_equalr+   repr)Znode1Znode2Z	children1Z	children2Zn1Zn2r   r   r   r6   y   s     

r6   c                 C   sL   t |  dd}t||}t||}dd l}d|jd|d|f S )NTkeependsr   zmThere's an issue with the diff parser. Please report (parso v%s) - Old/New:
%s
Actual Diff (May be empty):
%s )r   get_codedifflibZunified_diffparso__version__join)module	old_lines	new_linesZcurrent_linesZcurrent_diffZold_new_diffr=   r   r   r   _get_debug_error_message   s    rC   c                 C   sR   |   }t|r|jd S | }|jdkrDd|jv rD|jd d S |jd S d S )Nr   	endmarkerr$   r   )get_last_leaf_ends_with_newliner#   r   r   r)   r*   )Znode_or_leaf	last_leafnr   r   r   _get_last_line   s    
rI   c                 C   s*   | d ur&| j dkr&| jdkr&|  } q | S )Nr   r   )r   r   r   r   r   r   r   _skip_dedent_error_leaves   s    
rJ   r:   c                 C   s@   t | } | jdkr| j }n| j}|dkp>|dp>|dS )Nr   newliner$   r%   )rJ   r   r   lowerendswith)r   suffixtypr   r   r   rF      s
    
rF   c                 C   s   |D ]}|j dv r dS qdS )zg
    if, while, for and try might not be finished, because another part might
    still be parsed.
    )Zif_stmtZ
while_stmtZfor_stmtZtry_stmtFT)nonterminalpgen_grammarstackZ
stack_noder   r   r   _flows_finished   s    
rT   c                 C   sB   | j dkr| jd } | j dv r(| jd } | j dv o@| jd j dkS )N	decoratedr&   Zasync_funcdef
async_stmtZclassdefZfuncdefsuite)r   r   r   r   r   r   _func_or_class_has_suite   s
    



rZ   c                 C   sL   t | |sdS t|D ]0}|jdkr* dS |jdkrt|jdk  S qdS )NF	decoratorrY   r   T)rT   reversedrP   r+   nodesrQ   r   r   r   _suite_or_file_input_is_valid   s    


r^   c                 C   sB   | j dkr| jd } z| jd j}W n ty8   Y dS 0 |dv S )NrW   r   r   F)ifforwhiletrywith)r   r   r(   r'   )r   r(   r   r   r   _is_flow_node   s    

rd   c                   @   s   e Zd ZdS )_PositionUpdatingFinishedN)__name__
__module____qualname__r   r   r   r   re      s   re   c              	   C   sR   | D ]H}z
|j }W n, ty>   | j|7  _||u r:tY q0 t||| qd S r   )r   r'   r1   re   _update_positions)r]   line_offsetrG   r   r   r   r   r   ri      s    

ri   c                   @   sZ   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd ZdddZdS )
DiffParserz
    An advanced form of parsing a file faster. Unfortunately comes with huge
    side effects. It changes the given module.
    c                 C   s   || _ || _|| _d S r   )_pgen_grammar
_tokenizer_module)selfrR   Z	tokenizerr@   r   r   r   __init__   s    zDiffParser.__init__c                 C   s   d| _ d| _t| j| _d S Nr   )_copy_count_parser_count
_NodesTreern   _nodes_treero   r   r   r   _reset  s    zDiffParser._resetc              	   C   s  t d d| j_|| _|   t|}td|| j}|	 }t dt||f  |D ]\}}}}	}
t d||d ||	d |
 |
|kr|d dkr|
d8 }
|dkr|	| }| 
||d ||
 qZ|d	kr| j|
d
 qZ|dkr| j|
d
 qZ|dksZJ qZ| j  trzRd|}| j |ks0J t| j t| jdd| |}t| j| W n( ty   tt| j||  Y n0 | jjd }||krtd||f t| j|| t d | jS )a  
        The algorithm works as follows:

        Equal:
            - Assure that the start is a newline, otherwise parse until we get
              one.
            - Copy from parsed_until_line + 1 to max(i2 + 1)
            - Make sure that the indentation is correct (e.g. add DEDENT)
            - Add old and change positions
        Insert:
            - Parse from parsed_until_line + 1 to min(j2 + 1), hopefully not
              much more.

        Returns the new module node.
        zdiff parser startNzline_lengths old: %s; new: %sz!-> code[%s] old[%s:%s] new[%s:%s]r   r&   r:   equalreplace)
until_lineinsertdeleteTZerror_recoveryr   z(%s != %s) zdiff parser end)LOGdebugrn   Z_used_names_parser_lines_newrw   r+   r<   ZSequenceMatcherZget_opcodes_copy_from_old_parser_parseru   closeDEBUG_DIFF_PARSERr?   r;   r/   r   rl   parserm   r6   AssertionErrorprintrC   r*   	Exception)ro   rA   rB   line_lengthsmZopcodesZ	operationi1i2Zj1Zj2rj   codeZwithout_diff_parser_moduleZlast_posr   r   r   update
  s^    






zDiffParser.updatec                 C   s2   | j  d|kr.tdd|d| d S )Nr:   zparser issue:
%s
%s)rn   r;   r?   r~   warning)ro   rA   Z	lines_newr   r   r   _enabled_debuggingV  s    zDiffParser._enabled_debuggingc                 C   s  d}|| j jkr| j j| }| |d }|d u rH| | j jd  n|jj}||}	|dkr||d  j	t
r|g }
n$| j jd }| j ||	d  ||}
|
r|  jd7  _| j j}td|
d jd |
d jd d || n| | j jd  || j jksJ || j j}qd S )Nr&   r   r   zcopy old[%s:%s] new[%s:%s])ru   parsed_until_line_get_old_line_stmtr   r.   r   indexget_first_leafr)   r,   r   
copy_nodesrr   r~   r   r#   r*   )ro   rj   Zstart_line_oldZuntil_line_oldZuntil_line_newlast_until_lineZparsed_until_line_oldZ	line_stmtZ
p_childrenr   Zcopied_nodesZfrom_tor   r   r   r   Z  s8    

z DiffParser._copy_from_old_parserc                 C   sb   | j j|dfdd}t|r$| }| d |kr^|}|jjdvrL|j}q8|jd |kr^|S d S )Nr   T)Zinclude_prefixes)
file_inputrY   )rn   Zget_leaf_for_positionrF   r   r-   r.   r   r#   )ro   Zold_liner   r   r   r   r   r     s    zDiffParser._get_old_line_stmtc                 C   s   d}|| j jkr| |}|j}| j || j | jdurH| j| j jd< t	d|d 
 d | j j|jd d  || j jksJ || j j}qdS )zy
        Parses at least until the given line, but might just parse more until a
        valid state is reached.
        r   Nr&   z/parse_part from %s to %s (to %s in part parser)r   )ru   r   _try_parse_partr   add_parsed_nodes_keyword_token_indents_replace_tos_indentindentsr~   r   r-   r*   )ro   rz   r   r   r]   r   r   r   r     s    

	zDiffParser._parsec                 C   sR   |  j d7  _ | jj}| j|d }| j|||d}t| jdd| _| jj|dS )z
        Sets up a normal parser that uses a spezialized tokenizer to only parse
        until a certain position (or a bit longer if the statement hasn't
        ended.
        r   N)rj   Tr}   )tokens)	rs   ru   r   r   _diff_tokenizer   rl   _active_parserr   )ro   rz   r   Zlines_afterr   r   r   r   r     s    zDiffParser._try_parse_partr   c                 c   s  d}| j j}t|}| j||d df||dkd}| jj}d | _i | _|D ]T}	|	j}
|
t	krt||k rtt
| \}
}}}}	|
t	tfv r|
tkr|d d | _qnqqnd|v sd|v rtdd|}n4|d t|ksJ t||d t| dkrd}ttd||V   qn^|
tkr6|	jd |kr6d	}n>|rtd}t||krtt| j|rtttd|	jdV   q|
tkr|	jd
v rt|| j|	j< |	V  qJd S )NFr   r   )r#   r   Zis_first_tokenr$   r%   z
[^\n\r]+\Zr:   T)classdef)ru   r   r+   rm   r   rS   r   r   r   r   nextr
   resubr7   r   	ENDMARKERNEWLINEr#   r^   rl   NAMEstringlist)ro   linesrz   rj   Zwas_newliner   Zinitial_indentation_countr   rS   tokenrO   r   r#   r)   r   r   r   r     sX    


zDiffParser._diff_tokenizeN)r   )rf   rg   rh   __doc__rp   rw   r   r   r   r   r   r   r   r   r   r   r   rk      s   L,rk   c                   @   sJ   e Zd ZeddZdddZdd Zd	d
 ZdddZdd Z	dd Z
dS )_NodesTreeNode_ChildrenGroupz1prefix children line_offset last_line_offset_leafNr   c                 C   s"   || _ g | _|| _g | _|| _d S r   )r!   _children_groupsr.   _node_childrenindentation)ro   r!   r.   r   r   r   r   rp   
  s
    z_NodesTreeNode.__init__c           	   	   C   s   g }| j D ]\\}}}}t|d  }||j |_|dkr^zt||| W n ty\   Y n0 ||7 }q
|| j_|D ]}| j|_qt| j	D ]}|
  qd S rq   )r   r   r   r)   ri   re   r!   r   r.   r   finish)	ro   r   r)   Zchildren_partrj   last_line_offset_leafZ
first_leafr   Z
node_childr   r   r   r     s&    



z_NodesTreeNode.finishc                 C   s   | j | d S r   )r   append)ro   
child_noder   r   r   add_child_node(  s    z_NodesTreeNode.add_child_nodec                 C   s4   |d u r|d   }| ||||}| j| d S Nr&   )rE   r   r   r   )ro   r)   r   rj   r   groupr   r   r   add_tree_nodes+  s    z_NodesTreeNode.add_tree_nodesc                 C   s   d}| j r@| j d }t|j}|jd |j }t||r@|d8 }|tt|d 7 }|rt|dst|dst|d7 }| j	rt
|| j	d |S |S )Nr   r&   r   r$   r%   )r   r   r   r*   rj   rF   r+   r   rM   r   maxget_last_line)ro   rN   r1   Zchildren_grouprG   r   r   r   r   4  s    

z_NodesTreeNode.get_last_linec                 C   s   d| j j| jf S )Nz<%s: %s>)	__class__rf   r!   rv   r   r   r   __repr__L  s    z_NodesTreeNode.__repr__)Nr   )r   N)rf   rg   rh   r   r   rp   r   r   r   r   r   r   r   r   r   r     s   
  
	r   c                   @   sb   e Zd Zdd Zedd Zdd Zdd Zd	d
 Zdd Z	dd Z
dd ZdddZdd ZdS )rt   c                 C   s2   t || _| jg| _|| _d| _d| _dg| _d S )Nr:   r   )r   
_base_node_working_stackrn   _prefix_remainderr)   r   )ro   r@   r   r   r   rp   Q  s    

z_NodesTree.__init__c                 C   s   | j d | jS r   )r   r   r)   rv   r   r   r   r   Y  s    z_NodesTree.parsed_until_linec                 C   sB   t t| jD ].}|j|k s*|| jd u r2|  S | j  qd S rq   )r\   r   r   r   pop)ro   r   r   r   r   r   _update_insertion_node]  s    z!_NodesTree._update_insertion_nodec                 C   sz   | j }| |}|s$|| j  | _ d S |d jdks6J | |d jd }|jjdv sZJ ||| | |d | d S )Nr   rK   r   )rY   r   r&   )r)   _remove_endmarkerr   r   r#   r!   r   _update_parsed_node_tos)ro   
tree_nodeskeyword_token_indents
old_prefixr   r   r   r   r   c  s    
z_NodesTree.add_parsed_nodesc                 C   s   |j dkrn|jjd }t|||j d d}|dt|j | jd | | j	| | 
|jd | nt|r| 
|jd | d S )NrY   r   r&   r   r:   )r   r.   r   r   r#   r   r   r   r   r   r   rZ   )ro   r!   r   Zdef_leafZnew_tosr   r   r   r   r  s    
z"_NodesTree._update_parsed_node_tosc                 C   s   |d   }|jdk}d| _|rn|j}t|d|d}|dkrn|jd|d  |j|d d  |_| _d| _|r|j| _|dd }|S )zE
        Helps cleaning up the tree nodes that get inserted.
        r&   rD   r:   r$   r%   Nr   )rE   r   r   r)   r   rfind)ro   r   rG   Zis_endmarkerr)   Z
separationr   r   r   r     s    
"z_NodesTree._remove_endmarkerc                 c   sd   t |}|rt|V  t|}t|}|s8|| jvr8d S |V  |D ]}t||krX d S |V  qBd S r   )iterr   r   r   )ro   r   is_new_suiteZnode_iteratorZ
first_nodeindentrH   r   r   r   _get_matching_indent_nodes  s    
z%_NodesTree._get_matching_indent_nodesc           	         s   |d j dv rg S t|d  t| j}| j}| j} fdd| jD | _|   | t| j|||| j\}| _| _}|r|  j|7  _n|| _|| _|| _|S )zy
        Copies tree nodes from the old parser tree.

        Returns the number of tree nodes that were copied.
        r   r   
error_nodec                    s   g | ]}| kr|qS r   r   ).0ir   r   r   
<listcomp>      z)_NodesTree.copy_nodes.<locals>.<listcomp>)r   r   r   r   r)   r   r   _copy_nodes)	ro   r   rz   rj   Zold_working_stackr   Zold_indents	new_nodesadded_indentsr   r   r   r     s*    

z_NodesTree.copy_nodesr:   Fc              	   C   sF  g }g }t | j||d}d}	|D ]}
|
jd |kr: q|
jdkrJ q|
jdkrd|
jdv rd qt|
|krt|
r||
  qz
|
j}W n t	y   Y n^0 |
}|jdkr|jd }|jd	v r|jd }|jd
v r|jd }n|d }|jdv r q||
 q"|r|r|d }|jdv s<t
|d rzd}	|  |r|d }| jdkrjq|  qHqt|dkr|d jdkr|  qqq|sg |||fS |d }|d }d}t|r|}|jdkr|jd }qt|}|| t|t|d}| j||g |j||dd\}}}	}||7 }t|dk rd|  d}	n|snJ || |}d}|r:t|d  s|s|d  j}t|ddd }	|r|d }|jdkr|jd }|jd	v r|jd }|jd  }|dks J n|d  }||||| |	}d| _||||fS )N)r   r:   r   rD   r   )r   r
   rU   r&   rV   rX   r   rK   r   r   FrY   r   T)	is_nested   r8   :)r   r   r#   r   r   rI   rZ   r   r   r'   rd   r   rE   r+   r"   r   r   r   r   rF   r   r)   r   r   r   )ro   Zworking_stackr]   rz   rj   r)   r   r   r   
new_prefixr   crH   Z
suite_nodeZ	last_nodeZtosZhad_valid_suite_lastrY   r   Z	suite_tosZsuite_nodesZnew_working_stackZaiplastr   r   r   r   r     s    








	





z_NodesTree._copy_nodesc                 C   s
  | j   z| j }W n ty2   ddg}Y n0 t|}t|j}t| j	}t
|dks`J t
|dkr|d tr|ddgkr|d  d8  < |d  t
|d 7  < n(|d  t
|d 7  < t
|d |d< tdt|| j	| j }| j|_| jj| d S )Nr   r   r&   r:   )r   r   rn   rE   
IndexErrorrJ   r   r*   r   r)   r+   r,   r   r   tupler   r.   r   r   )ro   rG   r*   r   rD   r   r   r   r   _  s$    


z_NodesTree.closeN)r:   F)rf   rg   rh   rp   propertyr   r   r   r   r   r   r   r   r   r   r   r   r   rt   P  s   
# 
 rt   )r:   )0r   r   r<   collectionsr   loggingZparso.utilsr   Zparso.python.parserr   Zparso.python.treer   Zparso.python.tokenizer   r   Zparso.python.tokenr   	getLoggerrf   r~   r   r   r   r   r   r
   r   r   r   r   r"   r   r/   r6   rC   rI   rJ   rF   rT   rZ   r^   rd   r   re   ri   rk   r   rt   r   r   r   r   <module>   sN   
.

  K