a
    ,=icF                     @   s   d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlZddl	Z	ddl
mZ ddlmZ ddlmZ e	Zed	d
ZejejejejejfZG dd deZdd ZdddZdS )zToken generator for analyzing source code in logical units.

This module contains the TokenGenerator used for annotating a parsed syntax tree
with source code formatting.
    )absolute_import)division)print_functionN)StringIO)
formatting)fstring_utilsToken)typesrcstartendlinec                   @   s   e Zd ZdZd:ddZdd Zdd Zd	d
 Zdd Zdd Z	dd Z
d;ddZd<ddZd=ddZdd Zdd Zd>ddZd?d"d#Zd$d% Zd&d' Zejd@d(d)Zd*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 Zd4d5 Zd6d7 ZdAd8d9ZdS )BTokenGeneratora  Helper for sequentially parsing Python source code, token by token.

  Holds internal state during parsing, including:
  _tokens: List of tokens in the source code, as parsed by `tokenize` module.
  _parens: Stack of open parenthesis at the current point in parsing.
  _hints: Number of open parentheses, brackets, etc. at the current point.
  _scope_stack: Stack containing tuples of nodes where the last parenthesis that
    was open is related to one of the nodes on the top of the stack.
  _lines: Full lines of the source code.
  _i: Index of the last token that was parsed. Initially -1.
  _loc: (lineno, column_offset) pair of the position in the source that has been
     parsed to. This should be either the start or end of the token at index _i.

  Arguments:
    ignore_error_tokens: If True, will ignore error tokens. Otherwise, an error
      token will cause an exception. This is useful when the source being parsed
      contains invalid syntax, e.g. if it is in an fstring context.
  Fc                 C   sN   | d| _tt||| _g | _d| _g | _t| j| _	d| _
|  | _d S )NTr   )
splitlineslineslist_generate_tokens_tokens_parens_hints_scope_stacklen_len_i	loc_begin_loc)selfsourceignore_error_token r    k/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/pasta/base/token_generator.py__init__>   s    zTokenGenerator.__init__c                 C   s   t | d| j| j jS )N   r   )r   _space_betweenr   r   r   r   r    r    r!   chars_consumedH   s    zTokenGenerator.chars_consumedc                 C   s   | j dk rdS | j| j  jS )z7Get the start column of the current location parsed to.r   r#   )r   r   r   r&   r    r    r!   r   K   s    
zTokenGenerator.loc_beginc                 C   s   | j dk rdS | j| j  jS )z5Get the end column of the current location parsed to.r   r#   )r   r   r   r&   r    r    r!   loc_endQ   s    
zTokenGenerator.loc_endc                 C   s$   | j d | jkrdS | j| j d  S )z%Get the next token without advancing.r$   N)r   r   r   r&   r    r    r!   peekW   s    zTokenGenerator.peekc                 C   s   |  dd S )z4Get the next non-whitespace token without advancing.c                 S   s
   | j tvS N)r	   FORMATTING_TOKENStr    r    r!   <lambda>_       z4TokenGenerator.peek_non_whitespace.<locals>.<lambda>)peek_conditionalr&   r    r    r!   peek_non_whitespace]   s    z"TokenGenerator.peek_non_whitespacec                    s(   t  fdd| j| jd d D dS )z7Get the next token of the given type without advancing.c                 3   s   | ]} |r|V  qd S r*   r    ).0r-   	conditionr    r!   	<genexpr>c   r/   z2TokenGenerator.peek_conditional.<locals>.<genexpr>r$   N)nextr   r   )r   r4   r    r3   r!   r0   a   s    zTokenGenerator.peek_conditionalTc                 C   s>   |  j d7  _ | j | jkrdS |r2| j| j  j| _| j| j  S )zCConsume the next token and optionally advance the current location.r$   N)r   r   r   r   r   )r   advancer    r    r!   r6   e   s    zTokenGenerator.nextr$   c                 C   s   |  j |8  _ dS )zRewind the token iterator.N)r   )r   amountr    r    r!   rewindn   s    zTokenGenerator.rewindNc                    s     } s |r |jtjkr dS  fdd}tj|dd}  }d}t||r^|fndD ]<}|j	|j
7 }||kr||j7 }|j_	qd|j
_	qddu sdkr|r|jtjtjfv r| j7 }|S )	a+  Parses whitespace from the current _loc to the next non-whitespace.

    Arguments:
      max_lines: (optional int) Maximum number of lines to consider as part of
        the whitespace. Valid values are None, 0 and 1.
      comment: (boolean) If True, look for a trailing comment even when not in
        a parenthesized scope.

    Pre-condition:
      `_loc' represents the point before which everything has been parsed and
      after which nothing has been parsed.
    Post-condition:
      `_loc' is exactly at the character that was parsed to.
     c                    sX   | j tjtjfv pV| j tjkr( pVjpV| j tjkr>| jdkpVd u oV| j tjtj	fv S )N )
r	   TOKENSINDENTDEDENTCOMMENTr   
ERRORTOKENr
   NLNEWLINEtokencomment	max_linesr   r    r!   	predicate   s    z,TokenGenerator.whitespace.<locals>.predicateFr7   r    Nr   )r)   r	   r<   r?   r   	takewhile	itertoolschainr%   r   r   r
   r   rA   rB   r6   )r   rG   rF   Z
next_tokenrH   
whitespaceresulttokr    rE   r!   rM   r   s,    


zTokenGenerator.whitespacec                    s   | j }| jdd}s|S || _ |d}z&tfddttt|D }W n$ tyt   | j| j  j	| _
Y dS 0 |d|d  }| j| j  j	d d t|  t|  fd	d
 | j| j  j	| _
d|S )z@Parses whitespace from the current _loc to the end of the block.T)rF   c                 3   s$   | ]\}}|  d  r|V  qdS )#N)
startswith)r2   ir   )indent_levelr    r!   r5      s   
z2TokenGenerator.block_whitespace.<locals>.<genexpr>r:   Nr$   r   c                    s   | j d  k S )Nr   )r   rO   )end_liner    r!   r.      r/   z1TokenGenerator.block_whitespace.<locals>.<lambda>)r   rM   r   r6   reversedr   	enumerateStopIterationr   r   r   r   rJ   join)r   rS   start_iZfull_whitespacer   Zlast_line_idxr    )rU   rS   r!   block_whitespace   s     
&zTokenGenerator.block_whitespacec                 C   s`   d}d}| j }||k r\|  }|jdv s,J || ||j|j 7 }||jd7 }| j }q|S )zParse a number of dots.
    
    This is to work around an oddity in python3's tokenizer, which treats three
    `.` tokens next to each other in a FromImport's level as an ellipsis. This
    parses until the expected number of dots have been seen.
    r:   r   ).z...r\   )r   r6   r
   r%   r   count)r   Znum_dotsrN   Z	dots_seenprev_locrO   r    r    r!   dots   s    zTokenGenerator.dotsc                 C   s  d}g }| j }| j }}| dd D ]p}|| ||j7 }|jdkrb|rb|rb|   |j| _ q||j7 }|jdkr|| d}| j }| j}| j}q&|r|  }	|d  || | j|	j 7  < |	j| _|D ] }
| j	|
 | j
t| qn|| _ || _dS )z-Open a parenthesized scope on the given node.r:   c                 S   s   | j tv p| jdkS )N(r	   r+   r
   r,   r    r    r!   r.      r/   z+TokenGenerator.open_scope.<locals>.<lambda>r`   r   N)r   r   rJ   r%   r   r
   r9   appendr)   r   r   _scope_helper)r   nodesingle_parenrN   parensrZ   	start_locr^   rO   Znext_tokZparenr    r    r!   
open_scope   s8    



 zTokenGenerator.open_scopeprefixsuffixc                    sn  t ||du rt ||d t ||du r<t ||d | jrP|| jd vrTdS dh |rh d | j}| j }}d}	d}
|  fddD ]}|
| 	||j
7 }
|jdkr|r|	r|   | j}|j
}t |||
  q^|
|j7 }
|jdkrVd	}	| j  t ||| j  t |||
 d}
| j}|j}| jrP|| jd vrV q^|j}q|| _|| _dS )
z>Close a parenthesized scope on the given node, if one is open.Nr:   r   ),Fc                    s   | j tv p| j v S r*   ra   r,   symbolsr    r!   r.     r/   z,TokenGenerator.close_scope.<locals>.<lambda>T)fmtgetsetr   r   addr   r   rJ   r%   r   r
   r9   rb   popprependr   )r   rd   prefix_attrsuffix_attrtrailing_commare   Zparsed_to_iZparsed_to_locr^   Zencountered_parenrN   rO   r    rm   r!   close_scope   sJ    




zTokenGenerator.close_scopec                 C   s   |  j d7  _ dS )z5Indicates opening a group of parentheses or brackets.r$   N)r   r&   r    r    r!   	hint_open%  s    zTokenGenerator.hint_openc                 C   s$   |  j d8  _ | j dk r tddS )z5Indicates closing a group of parentheses or brackets.r$   r   zHint value negativeN)r   
ValueErrorr&   r    r    r!   hint_closed)  s    
zTokenGenerator.hint_closedc                 c   sL   | j ||dud dV  |r:| j||d |d |dd n| j||d dS )z0Context manager to handle a parenthesized scope.N)re   _prefixZ_suffixT)ru   rv   rw   re   )rw   )rh   rx   )r   rd   attrrw   r    r    r!   scope/  s    zTokenGenerator.scopec                 C   s   | j p
| jS )z&Return True iff there is a scope open.)r   r   r&   r    r    r!   is_in_scope<  s    zTokenGenerator.is_in_scopec                    s    fdd}  |S )z+Parse a full string literal from the input.c                    s,   | j tjtjfv p*  o*| j tjtjfv S r*   )r	   r<   STRINGr?   r   rA   rB   rC   r&   r    r!   rH   B  s    z%TokenGenerator.str.<locals>.predicate)
eat_tokens)r   rH   r    r&   r!   str@  s    zTokenGenerator.strc                 C   sT   d}| j }d}| j|ddD ]&}|| ||j7 }||j7 }|j}q|rP|j| _ |S )z7Parse input from tokens while a given condition is met.r:   NFrI   )r   rJ   r%   r   r
   r   )r   rH   contentr^   rO   r    r    r!   r   H  s    
zTokenGenerator.eat_tokensc                    s    fdd}|S )a  Parses an fstring, including subexpressions.

    Returns:
      A generator function which, when repeatedly reads a chunk of the fstring
      up until the next subexpression and yields that chunk, plus a new token
      generator to use to parse the subexpression. The subexpressions in the
      original fstring data are replaced by placeholders to make it possible to
      fill them in with new values, if desired.
    c            
      3   s:    jtjkr } n fdd d _ } t| }d}d}d}|t| d k r,t	|\}}||7 }|dkrJt	|\}}|dkr||7 }qJt
||fg|}|t|d 7 }|d7 }t| |d d  d	d
}||fV  d}t| D ]}	t	|\}}qt	|\}}|dkrJt	|\}}qqJ|d fV  d S )Nc                    sZ   | j tjkr2| jdkr2 jdkr$dS   jd8  _| j tjkrV| jdkrV  jd7  _dS )N}r   Fr$   {T)r	   r<   OPr
   levelrT   
fstr_eaterr    r!   r   k  s    
z<TokenGenerator.fstr.<locals>.fstr_parser.<locals>.fstr_eaterr   r   r:   r$   r   r   T)r   )r1   r	   r<   r   r   r   r   rW   r   r6   rK   rL   r   placeholderr   ranger'   )
Zstr_contentZindexed_charsZval_idxrR   rN   cZnextiZnextctgZtg_ir&   r   r!   fstr_parser`  s:    


z(TokenGenerator.fstr.<locals>.fstr_parserr    )r   r   r    r&   r!   fstrV  s    
;zTokenGenerator.fstrc              	   C   s   ||krt d|||d t| jkr*dS |\}}|\}}||krX| j|d  || S dt| j|d  |d f| j||d  |dkr| j|d  d| fndS )z5Parse the space between a location and the next tokenzstart_loc > end_locr   r:   r$   N)rz   r   r   rY   rK   rL   )r   rg   Zend_locprev_rowprev_colZend_rowZend_colr    r    r!   r%     s    "zTokenGenerator._space_betweenc                 C   s6   | j }dd }t| j|dd}| jdd}|| _ |S )zParse the next name token.c                 S   s   | j tjkS r*   )r	   r<   NAMErC   r    r    r!   rH     s    z+TokenGenerator.next_name.<locals>.predicateFrI   )r   r   rJ   r6   )r   Zlast_irH   Zunused_tokensrN   r    r    r!   	next_name  s    zTokenGenerator.next_namec                 C   sH   |   }|j|krDtdtj| |j|jd | j|jd d  f |S )z.Parse a token of the given type and return it.z$Expected %r but found %r
line %d: %sr   r$   )r6   r	   rz   tokenizetok_namer
   r   r   )r   
token_typerD   r    r    r!   next_of_type  s    
zTokenGenerator.next_of_typec                 c   sN   | j }| j|d}|dur<||r<|V  | j }| j|d}q|   || _ dS )z<Parse tokens as long as a condition holds on the next token.rI   N)r   r6   r9   )r   r4   r7   r^   rD   r    r    r!   rJ     s    zTokenGenerator.takewhile)F)T)r$   )NF)F)ri   rj   FF)NF)T)__name__
__module____qualname____doc__r"   r'   r   r(   r)   r1   r0   r6   r9   rM   r[   r_   rh   rx   ry   r{   
contextlibcontextmanagerr~   r   r   r   r   r%   r   r   rJ   r    r    r    r!   r   *   s:   


	

+
)  
2G	r   c                 C   sr  t | tjr| ft| j S t | tjr8| ft| j S t | tjrX| ft| jd  S t | tjrt| ft| j	 S t | tj
r| ft| j S t | tjr| ft| j S t | tjr| ft| jd  S t | tjr| ft| j S t | tjr| jr| ft| jd  S t | tjr0| ft| j S t | tjrN| ft| j S t | tjrl| ft| j S | fS )a  Get the closure of nodes that could begin a scope at this point.

  For instance, when encountering a `(` when parsing a BinOp node, this could
  indicate that the BinOp itself is parenthesized OR that the BinOp's left node
  could be parenthesized.

  E.g.: (a + b * c)   or   (a + b) * c   or   (a) + b * c
        ^                  ^                  ^

  Arguments:
    node: (ast.AST) Node encountered when opening a scope.

  Returns:
    A closure of nodes which that scope might apply to.
  r   )
isinstanceast	Attributerc   value	SubscriptAssigntargets	AugAssigntargetExprCompareleftBoolOpvaluesBinOpTupleeltsCallfuncGeneratorExpeltIfExpbody)rd   r    r    r!   rc     s2    rc   Fc                 c   sH   t t| j}z|D ]}t| V  qW n t jyB   |s> Y n0 d S r*   )r   generate_tokensr   readliner   
TokenError)r   r   Ztoken_generatorrO   r    r    r!   r     s    r   )F)r   
__future__r   r   r   r   collectionsr   rK   r   sixr   Z
pasta.baser   ro   r   r<   
namedtupler   r=   r>   rA   rB   r?   r+   objectr   rc   r   r    r    r    r!   <module>   s,      (+