"""DEPRECATED: Input transformer classes to support IPython special syntax. This module was deprecated in IPython 7.0, in favour of inputtransformer2. This includes the machinery to recognise and transform ``%magic`` commands, ``!system`` commands, ``help?`` querying, prompt stripping, and so forth. """ import abc import functools import re import tokenize from tokenize import generate_tokens, untokenize, TokenError from io import StringIO from IPython.core.splitinput import LineInfo #----------------------------------------------------------------------------- # Globals #----------------------------------------------------------------------------- # The escape sequences that define the syntax transformations IPython will # apply to user input. These can NOT be just changed here: many regular # expressions and other parts of the code may use their hardcoded values, and # for all intents and purposes they constitute the 'IPython syntax', so they # should be considered fixed. ESC_SHELL = '!' # Send line to underlying system shell ESC_SH_CAP = '!!' # Send line to system shell and capture output ESC_HELP = '?' # Find information about object ESC_HELP2 = '??' # Find extra-detailed information about object ESC_MAGIC = '%' # Call magic function ESC_MAGIC2 = '%%' # Call cell-magic function ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call ESC_QUOTE2 = ';' # Quote all args as a single string, call ESC_PAREN = '/' # Call first argument with rest of line as arguments ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\ ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\ ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ] class InputTransformer(metaclass=abc.ABCMeta): """Abstract base class for line-based input transformers.""" @abc.abstractmethod def push(self, line): """Send a line of input to the transformer, returning the transformed input or None if the transformer is waiting for more input. Must be overridden by subclasses. Implementations may raise ``SyntaxError`` if the input is invalid. No other exceptions may be raised. """ pass @abc.abstractmethod def reset(self): """Return, transformed any lines that the transformer has accumulated, and reset its internal state. Must be overridden by subclasses. """ pass @classmethod def wrap(cls, func): """Can be used by subclasses as a decorator, to return a factory that will allow instantiation with the decorated object. """ @functools.wraps(func) def transformer_factory(**kwargs): return cls(func, **kwargs) return transformer_factory class StatelessInputTransformer(InputTransformer): """Wrapper for a stateless input transformer implemented as a function.""" def __init__(self, func): self.func = func def __repr__(self): return "StatelessInputTransformer(func={0!r})".format(self.func) def push(self, line): """Send a line of input to the transformer, returning the transformed input.""" return self.func(line) def reset(self): """No-op - exists for compatibility.""" pass class CoroutineInputTransformer(InputTransformer): """Wrapper for an input transformer implemented as a coroutine.""" def __init__(self, coro, **kwargs): # Prime it self.coro = coro(**kwargs) next(self.coro) def __repr__(self): return "CoroutineInputTransformer(coro={0!r})".format(self.coro) def push(self, line): """Send a line of input to the transformer, returning the transformed input or None if the transformer is waiting for more input. """ return self.coro.send(line) def reset(self): """Return, transformed any lines that the transformer has accumulated, and reset its internal state. """ return self.coro.send(None) class TokenInputTransformer(InputTransformer): """Wrapper for a token-based input transformer. func should accept a list of tokens (5-tuples, see tokenize docs), and return an iterable which can be passed to tokenize.untokenize(). """ def __init__(self, func): self.func = func self.buf = [] self.reset_tokenizer() def reset_tokenizer(self): it = iter(self.buf) self.tokenizer = generate_tokens(it.__next__) def push(self, line): self.buf.append(line + '\n') if all(l.isspace() for l in self.buf): return self.reset() tokens = [] stop_at_NL = False try: for intok in self.tokenizer: tokens.append(intok) t = intok[0] if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL): # Stop before we try to pull a line we don't have yet break elif t == tokenize.ERRORTOKEN: stop_at_NL = True except TokenError: # Multi-line statement - stop and try again with the next line self.reset_tokenizer() return None return self.output(tokens) def output(self, tokens): self.buf.clear() self.reset_tokenizer() return untokenize(self.func(tokens)).rstrip('\n') def reset(self): l = ''.join(self.buf) self.buf.clear() self.reset_tokenizer() if l: return l.rstrip('\n') class assemble_python_lines(TokenInputTransformer): def __init__(self): super(assemble_python_lines, self).__init__(None) def output(self, tokens): return self.reset() @CoroutineInputTransformer.wrap def assemble_logical_lines(): r"""Join lines following explicit line continuations (\)""" line = '' while True: line = (yield line) if not line or line.isspace(): continue parts = [] while line is not None: if line.endswith('\\') and (not has_comment(line)): parts.append(line[:-1]) line = (yield None) # Get another line else: parts.append(line) break # Output line = ''.join(parts) # Utilities def _make_help_call(target, esc, lspace): """Prepares a pinfo(2)/psearch call from a target name and the escape (i.e. ? or ??)""" method = 'pinfo2' if esc == '??' \ else 'psearch' if '*' in target \ else 'pinfo' arg = " ".join([method, target]) #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args) t_magic_name, _, t_magic_arg_s = arg.partition(' ') t_magic_name = t_magic_name.lstrip(ESC_MAGIC) return "%sget_ipython().run_line_magic(%r, %r)" % ( lspace, t_magic_name, t_magic_arg_s, ) # These define the transformations for the different escape characters. def _tr_system(line_info): "Translate lines escaped with: !" cmd = line_info.line.lstrip().lstrip(ESC_SHELL) return '%sget_ipython().system(%r)' % (line_info.pre, cmd) def _tr_system2(line_info): "Translate lines escaped with: !!" cmd = line_info.line.lstrip()[2:] return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd) def _tr_help(line_info): "Translate lines escaped with: ?/??" # A naked help line should just fire the intro help screen if not line_info.line[1:]: return 'get_ipython().show_usage()' return _make_help_call(line_info.ifun, line_info.esc, line_info.pre) def _tr_magic(line_info): "Translate lines escaped with: %" tpl = '%sget_ipython().run_line_magic(%r, %r)' if line_info.line.startswith(ESC_MAGIC2): return line_info.line cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip() #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args) t_magic_name, _, t_magic_arg_s = cmd.partition(' ') t_magic_name = t_magic_name.lstrip(ESC_MAGIC) return tpl % (line_info.pre, t_magic_name, t_magic_arg_s) def _tr_quote(line_info): "Translate lines escaped with: ," return '%s%s("%s")' % (line_info.pre, line_info.ifun, '", "'.join(line_info.the_rest.split()) ) def _tr_quote2(line_info): "Translate lines escaped with: ;" return '%s%s("%s")' % (line_info.pre, line_info.ifun, line_info.the_rest) def _tr_paren(line_info): "Translate lines escaped with: /" return '%s%s(%s)' % (line_info.pre, line_info.ifun, ", ".join(line_info.the_rest.split())) tr = { ESC_SHELL : _tr_system, ESC_SH_CAP : _tr_system2, ESC_HELP : _tr_help, ESC_HELP2 : _tr_help, ESC_MAGIC : _tr_magic, ESC_QUOTE : _tr_quote, ESC_QUOTE2 : _tr_quote2, ESC_PAREN : _tr_paren } @StatelessInputTransformer.wrap def escaped_commands(line): """Transform escaped commands - %magic, !system, ?help + various autocalls. """ if not line or line.isspace(): return line lineinf = LineInfo(line) if lineinf.esc not in tr: return line return tr[lineinf.esc](lineinf) _initial_space_re = re.compile(r'\s*') _help_end_re = re.compile(r"""(%{0,2} (?!\d)[\w*]+ # Variable name (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, re.VERBOSE) # Extra pseudotokens for multiline strings and data structures _MULTILINE_STRING = object() _MULTILINE_STRUCTURE = object() def _line_tokens(line): """Helper for has_comment and ends_in_comment_or_string.""" readline = StringIO(line).readline toktypes = set() try: for t in generate_tokens(readline): toktypes.add(t[0]) except TokenError as e: # There are only two cases where a TokenError is raised. if 'multi-line string' in e.args[0]: toktypes.add(_MULTILINE_STRING) else: toktypes.add(_MULTILINE_STRUCTURE) return toktypes def has_comment(src): """Indicate whether an input line has (i.e. ends in, or is) a comment. This uses tokenize, so it can distinguish comments from # inside strings. Parameters ---------- src : string A single line input string. Returns ------- comment : bool True if source has a comment. """ return (tokenize.COMMENT in _line_tokens(src)) def ends_in_comment_or_string(src): """Indicates whether or not an input line ends in a comment or within a multiline string. Parameters ---------- src : string A single line input string. Returns ------- comment : bool True if source ends in a comment or multiline string. """ toktypes = _line_tokens(src) return (tokenize.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes) @StatelessInputTransformer.wrap def help_end(line): """Translate lines with ?/?? at the end""" m = _help_end_re.search(line) if m is None or ends_in_comment_or_string(line): return line target = m.group(1) esc = m.group(3) lspace = _initial_space_re.match(line).group(0) return _make_help_call(target, esc, lspace) @CoroutineInputTransformer.wrap def cellmagic(end_on_blank_line=False): """Captures & transforms cell magics. After a cell magic is started, this stores up any lines it gets until it is reset (sent None). """ tpl = 'get_ipython().run_cell_magic(%r, %r, %r)' cellmagic_help_re = re.compile(r'%%\w+\?') line = '' while True: line = (yield line) # consume leading empty lines while not line: line = (yield line) if not line.startswith(ESC_MAGIC2): # This isn't a cell magic, idle waiting for reset then start over while line is not None: line = (yield line) continue if cellmagic_help_re.match(line): # This case will be handled by help_end continue first = line body = [] line = (yield None) while (line is not None) and \ ((line.strip() != '') or not end_on_blank_line): body.append(line) line = (yield None) # Output magic_name, _, first = first.partition(' ') magic_name = magic_name.lstrip(ESC_MAGIC2) line = tpl % (magic_name, first, u'\n'.join(body)) def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None): """Remove matching input prompts from a block of input. Parameters ---------- prompt_re : regular expression A regular expression matching any input prompt (including continuation) initial_re : regular expression, optional A regular expression matching only the initial prompt, but not continuation. If no initial expression is given, prompt_re will be used everywhere. Used mainly for plain Python prompts, where the continuation prompt ``...`` is a valid Python expression in Python 3, so shouldn't be stripped. Notes ----- If `initial_re` and `prompt_re differ`, only `initial_re` will be tested against the first line. If any prompt is found on the first two lines, prompts will be stripped from the rest of the block. """ if initial_re is None: initial_re = prompt_re line = '' while True: line = (yield line) # First line of cell if line is None: continue out, n1 = initial_re.subn('', line, count=1) if turnoff_re and not n1: if turnoff_re.match(line): # We're in e.g. a cell magic; disable this transformer for # the rest of the cell. while line is not None: line = (yield line) continue line = (yield out) if line is None: continue # check for any prompt on the second line of the cell, # because people often copy from just after the first prompt, # so we might not see it in the first line. out, n2 = prompt_re.subn('', line, count=1) line = (yield out) if n1 or n2: # Found a prompt in the first two lines - check for it in # the rest of the cell as well. while line is not None: line = (yield prompt_re.sub('', line, count=1)) else: # Prompts not in input - wait for reset while line is not None: line = (yield line) @CoroutineInputTransformer.wrap def classic_prompt(): """Strip the >>>/... prompts of the Python interactive shell.""" # FIXME: non-capturing version (?:...) usable? prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)') initial_re = re.compile(r'^>>>( |$)') # Any %magic/!system is IPython syntax, so we needn't look for >>> prompts turnoff_re = re.compile(r'^[%!]') return _strip_prompts(prompt_re, initial_re, turnoff_re) @CoroutineInputTransformer.wrap def ipy_prompt(): """Strip IPython's In [1]:/...: prompts.""" # FIXME: non-capturing version (?:...) usable? prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)') # Disable prompt stripping inside cell magics turnoff_re = re.compile(r'^%%') return _strip_prompts(prompt_re, turnoff_re=turnoff_re) @CoroutineInputTransformer.wrap def leading_indent(): """Remove leading indentation. If the first line starts with a spaces or tabs, the same whitespace will be removed from each following line until it is reset. """ space_re = re.compile(r'^[ \t]+') line = '' while True: line = (yield line) if line is None: continue m = space_re.match(line) if m: space = m.group(0) while line is not None: if line.startswith(space): line = line[len(space):] line = (yield line) else: # No leading spaces - wait for reset while line is not None: line = (yield line) _assign_pat = \ r'''(?P(\s*) ([\w\.]+) # Initial identifier (\s*,\s* \*?[\w\.]+)* # Further identifiers for unpacking \s*?,? # Trailing comma ) \s*=\s* ''' assign_system_re = re.compile(r'{}!\s*(?P.*)'.format(_assign_pat), re.VERBOSE) assign_system_template = '%s = get_ipython().getoutput(%r)' @StatelessInputTransformer.wrap def assign_from_system(line): """Transform assignment from system commands (e.g. files = !ls)""" m = assign_system_re.match(line) if m is None: return line return assign_system_template % m.group('lhs', 'cmd') assign_magic_re = re.compile(r'{}%\s*(?P.*)'.format(_assign_pat), re.VERBOSE) assign_magic_template = '%s = get_ipython().run_line_magic(%r, %r)' @StatelessInputTransformer.wrap def assign_from_magic(line): """Transform assignment from magic commands (e.g. a = %who_ls)""" m = assign_magic_re.match(line) if m is None: return line #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args) m_lhs, m_cmd = m.group('lhs', 'cmd') t_magic_name, _, t_magic_arg_s = m_cmd.partition(' ') t_magic_name = t_magic_name.lstrip(ESC_MAGIC) return assign_magic_template % (m_lhs, t_magic_name, t_magic_arg_s)