
^Yc           @   sn  d  Z  d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l m Z d d l m Z d d l m	 Z	 e j
 e  Z d Z e e j e j g  Z e j d g j  j e  d d k Z e e j e j e j e j g  Z d	 e f d
     YZ d   Z e r+e d  Z n  d   Z d   Z d   Z d   Z d   Z  d   Z! d   Z" d S(   sF   Module containing our file processor that tokenizes a file for checks.iN(   t   defaults(   t
   exceptions(   t   utilsi   s   #
i   t   FileProcessorc           B   s  e  Z d  Z d d  Z e d    Z e j d    Z	 d   Z
 d   Z d   Z d   Z d   Z d	   Z d
   Z d   Z d   Z d   Z d d  Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z RS(   s!  Processes a file and holdes state.

    This processes a file by generating tokens, logical and physical lines,
    and AST trees. This also provides a way of passing state about the file
    to checks expecting that state. Any public attribute on this object can
    be requested by a plugin. The known public attributes are:

    - :attr:`blank_before`
    - :attr:`blank_lines`
    - :attr:`checker_state`
    - :attr:`indent_char`
    - :attr:`indent_level`
    - :attr:`line_number`
    - :attr:`logical_line`
    - :attr:`max_line_length`
    - :attr:`multiline`
    - :attr:`noqa`
    - :attr:`previous_indent_level`
    - :attr:`previous_logical`
    - :attr:`previous_unindented_logical_line`
    - :attr:`tokens`
    - :attr:`file_tokens`
    - :attr:`total_lines`
    - :attr:`verbose`
    c         C   s  | |  _  | |  _ | |  _ | d k r9 |  j   |  _ n  |  j   d |  _ d |  _ i  |  _ d |  _	 | j
 |  _
 d |  _ d |  _ d |  _ d |  _ | j |  _ t |  _ t |  _ d |  _ d |  _ d |  _ g  |  _ t |  j  |  _ | j |  _ i d d 6|  _ d |  _ d S(   sm   Initialice our file processor.

        :param str filename:
            Name of the file to process
        i    t    s   logical linesN(   t   optionst   filenamet   linest   Nonet
   read_linest   strip_utf_bomt   blank_beforet   blank_linest   _checker_statest   checker_statet   hang_closingt   indent_chart   indent_levelt   line_numbert   logical_linet   max_line_lengtht   Falset	   multilinet   noqat   previous_indent_levelt   previous_logicalt    previous_unindented_logical_linet   tokenst   lent   total_linest   verboset
   statisticst   _file_tokens(   t   selfR   R   R   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   __init__3   s6    			
														c            s~   |  j  d k rw t |  j    y% t t j   f d     |  _  Wqw t j k
 rs } t j	 | j
 d |  qw Xn  |  j  S(   s   The complete set of tokens for a file.

        Accessing this attribute *may* raise an InvalidSyntax exception.

        :raises: flake8.exceptions.InvalidSyntax
        c              s
   t     S(   N(   t   next(    (   t	   line_iter(    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   <lambda>w   s    t	   exceptionN(   R    R   t   iterR   t   listt   tokenizet   generate_tokenst
   TokenErrorR   t   InvalidSyntaxt   message(   R!   t   exc(    (   R$   s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   file_tokensk   s    	c         c   s$   | |  _  t |  _ d Vt |  _ d S(   s2   Context-manager to toggle the multiline attribute.N(   R   t   TrueR   R   (   R!   R   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   inside_multiline~   s    		c         C   s   d |  _  d S(   s)   Reset the blank_before attribute to zero.i    N(   R   (   R!   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   reset_blank_before   s    c         C   s   |  j  d =d S(   s-   Delete the first token in the list of tokens.i    N(   R   (   R!   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   delete_first_token   s    c         C   s   |  j  d 7_  d S(   s&   Note that we visited a new blank line.i   N(   R   (   R!   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   visited_new_blank_line   s    c         C   s]   | d d \ } } |  j  | d } t | |   |  _ |  j |  j k  rY |  j |  _ n  d S(   s:   Update the indent level based on the logical line mapping.i    i   N(   R   t   expand_indentR   R   R   (   R!   t   mappingt	   start_rowt	   start_colt
   start_line(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   update_state   s
    c         C   s3   d | d k r/ |  j  j | d i   |  _ n  d S(   s2   Update the checker_state attribute for the plugin.R   t
   parameterst   nameN(   R   t
   setdefaultR   (   R!   t   plugin(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   update_checker_state_for   s    	c         C   s[   |  j  r< |  j |  _ |  j  |  _ |  j s< |  j  |  _ q< n  d |  _ g  |  _ t |  _ d S(   so   Record the previous logical line.

        This also resets the tokens list and the blank_lines count.
        i    N(	   R   R   R   R   R   R   R   R   R   (   R!   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   next_logical_line   s    				c         C   s  g  } g  } d } d } } } x`|  j D]U\ } } }	 }
 } | t k rQ q* n  | si d |	 f g } n  | t j k r | j |  q* n  | t j k r t |  } n  | rC|	 \ } } | | k r | d } | d } |  j | | } | d k s| d k r@| d k r@d | } q@qC| | k rC| | | !| } qCn  | j |  | t	 |  7} | j | |
 f  |
 \ } } q* W| | | f S(   s4   Build the mapping, comments, and logical line lists.i    i   t   ,s   {[(s   }])t    N(
   R   R   t   SKIP_TOKENSR)   t   COMMENTt   appendt   STRINGt   mutate_stringR   R   (   R!   t   logicalt   commentst   lengtht   previous_rowt   previous_columnR6   t
   token_typet   textt   startt   endt   lineR7   t   start_columnt	   row_indext   column_indext   previous_text(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   build_logical_line_tokens   s>    

c         C   s   t  d j |  j  d d t  S(   s5   Build an abstract syntax tree from the list of lines.R   t   exec(   t   compilet   joinR   t   PyCF_ONLY_AST(   R!   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt	   build_ast   s    c         C   sw   |  j    \ } } } d j |  } d j |  |  _ t j j |  rT t |  _ n  |  j d c d 7<| |  j | f S(   s2   Build a logical line from the current tokens list.R   s   logical linesi   (	   RV   RY   R   R    t   NOQA_INLINE_REGEXPt   searchR0   R   R   (   R!   RI   RH   t   mapping_listt   joined_comments(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   build_logical_line   s    c         c   s:   x3 | d j  d  d  D] } | V|  j d 7_ q Wd S(   s|   Split a physical line's line based on new-lines.

        This also auto-increments the line number for the caller.
        i   s   
iN(   t   splitR   (   R!   t   tokenRQ   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt
   split_line   s    c         C   s   | d k r i  } n  x | j   D]t \ } } | | k r@ q" n  y t |  |  | | <Wq" t k
 r } | r t j |    q t j d |  q" Xq" W| S(   s8   Generate the keyword arguments for a list of parameters.sP   Plugin requested optional parameter "%s" but this is not an available parameter.N(   R   t   itemst   getattrt   AttributeErrort   LOGR&   t   warning(   R!   R;   t	   argumentst   paramt   requiredR.   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   keyword_arguments_for   s    		c         C   s    | d k r | d |  _  n  d S(   s/   Update attributes based on error code and line.t   E101i    N(   R   (   R!   t
   error_codeRQ   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   check_physical_error   s    c         c   s   yQ xJ t  j |  j  D]6 } | d d |  j k r7 Pn  |  j j |  | Vq WWn. t  j t f k
 r } t j	 d |   n Xd S(   s   Tokenize the file and yield the tokens.

        :raises flake8.exceptions.InvalidSyntax:
            If a :class:`tokenize.TokenError` is raised while generating
            tokens.
        i   i    R&   N(
   R)   R*   t	   next_lineR   R   RE   R+   t   SyntaxErrorR   R,   (   R!   Rb   R.   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyR*      s    c         C   s>   | d } d | k o* t  |  j  k  n r: |  j | Sd S(   s8   Retrieve the physical line at the specified line number.i   i    N(   R   R   R   (   R!   R   t   adjusted_line_number(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   line_for  s    
%c         C   sk   |  j  |  j k r d S|  j |  j  } |  j  d 7_  |  j d k rg | d  t j k rg | d |  _ n  | S(   s    Get the next line from the list.R   i   i    N(   R   R   R   R   R   R    t
   WHITESPACE(   R!   RQ   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyRp     s    "c         C   sR   |  j  d k s |  j  d k rB |  j j p- d |  _  |  j   } n |  j   } | S(   s%   Read the lines for this file checker.t   -t   stdinN(   R   R   R   t   stdin_display_namet   read_lines_from_stdint   read_lines_from_filename(   R!   R   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyR	   $  s
    c         C   s)   t  |  j d   } | j   SWd  QXd  S(   Nt   rU(   t   openR   t	   readlines(   R!   t   fd(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   _readlines_py2.  s    c      	   C   s   yy t  |  j d  a } t j | j  \ } } t j | | d t } g  | D] } | j |  ^ qO | j	   SWd  QXWnB t
 t t f k
 r t  |  j d d  } | j	   SWd  QXn Xd  S(   Nt   rbt   line_bufferingt   encodings   latin-1(   R{   R   R)   t   detect_encodingt   readlinet   iot   TextIOWrapperR0   t   decodeR|   t   LookupErrorRq   t   UnicodeError(   R!   R}   t   codingR   t   textfdt   l(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   _readlines_py33  s    c         C   s]   d t  j k o d k  n r+ |  j } n+ d t  j k oE d	 k  n rV |  j } n  |   S(
   s   Read the lines for a file.i   i   i   i    i   (   i   i   (   i   i    (   i   i    (   i   i    (   t   syst   version_infoR~   R   (   R!   R|   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyRy   A  s
    c         C   s   t  j   j t  S(   s    Read the lines from standard in.(   R   t   stdin_get_valuet
   splitlinesR0   (   R!   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyRx   J  s    c            s)   t  j j   t   f d   |  j D  S(   s   Check if ``# flake8: noqa`` is in the file to be ignored.

        :returns:
            True if a line matches :attr:`defaults.NOQA_FILE`,
            otherwise False
        :rtype:
            bool
        c         3   s   |  ] }   |  Vq d  S(   N(    (   t   .0RQ   (   t   ignore_file(    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pys	   <genexpr>Z  s    (   R    t	   NOQA_FILER]   t   anyR   (   R!   (    (   R   s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   should_ignore_fileO  s    
c         C   s   |  j  s d St |  j  d d  } | d k r4 d S| d k r[ |  j  d d |  j  d <n2 |  j  d d  d k r |  j  d d |  j  d <n  d S(	   s-   Strip the UTF bom from the lines of the file.Ni    i   i  i   i   s   ﻿(   i   i  (   R   t   ord(   R!   t
   first_byte(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyR
   \  s    	N(    t   __name__t
   __module__t   __doc__R   R"   t   propertyR/   t
   contextlibt   contextmanagerR1   R2   R3   R4   R:   R?   R@   RV   R[   R`   Rc   Rl   Ro   R*   Rs   Rp   R	   R~   R   Ry   Rx   R   R
   (    (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyR      s4   8							"		
					
	
	
						c         C   s0   |  d t  k p/ |  d |  d d j   d k S(   s+   Check if the token is an end-of-line token.i    i   i   i   s   \
(   t   NEWLINEt   lstrip(   Rb   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   is_eol_tokenn  s    c         C   s1   | |   p0 |  d t  j k o0 |  d |  d k S(   s+   Check if the token is an end-of-line token.i    i   i   (   R)   RD   (   Rb   t   _is_eol_token(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyR   t  s    c         C   s!   |  d t  j k o  d |  d k S(   s$   Check if this is a multiline string.i    s   
i   (   R)   RF   (   Rb   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   is_multiline_stringz  s    c         C   s   |  d t  k S(   s0   Check if the token type is a newline token type.i    (   R   (   Rb   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   token_is_newline  s    c         C   s   t  o |  d t j k S(   s%   Check if the token type is a comment.i    (   t   COMMENT_WITH_NLR)   RD   (   Rb   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   token_is_comment  s    c         C   s8   |  p	 d }  | d k r  |  d S| d k r4 |  d S|  S(   s    Count the number of parentheses.i    s   ([{i   s   }])(    (   t   current_parentheses_countt
   token_text(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   count_parentheses  s    c         C   s   | d d | d d k rE d | d d p0 d | d d f } n d | d d } |  j  t j d | d d | t j | d | d f  d	 S(
   s)   Log a token to a provided logging object.i   i    i   s   [%s:%s]i   R   s   l.%ss   l.%s	%s	%s	%rN(   t   logt   flake8t   _EXTRA_VERBOSER)   t   tok_name(   R   Rb   t   pos(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt	   log_token  s    )c         C   s|   d |  k r& t  |   t  |  j    Sd } xI |  D]A } | d k rZ | d d d } q3 | d k rs | d 7} q3 Pq3 W| S(   s   Return the amount of indentation.

    Tabs are expanded to the next multiple of 8.

    >>> expand_indent('    ')
    4
    >>> expand_indent('\t')
    8
    >>> expand_indent('       \t')
    8
    >>> expand_indent('        \t')
    16
    s   	i    i   RB   i   (   R   R   (   RQ   t   resultt   char(    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyR5     s    c         C   sj   |  j  |  d  d } t |   d } |  d d k rN | d 7} | d 8} n  |  |  d | | |  | S(	   s   Replace contents with 'xxx' to prevent syntax matching.

    >>> mute_string('"abc"')
    '"xxx"'
    >>> mute_string("'''abc'''")
    "'''xxx'''"
    >>> mute_string("r'abc'")
    "r'xxx'"
    ii   is   """s   '''i   t   x(   s   """s   '''(   t   indexR   (   RN   RO   RP   (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyRG     s    
(#   R   R   R   t   loggingR   R)   R   R    R   R   t	   getLoggerR   Rg   RZ   t	   frozensett   NLR   R*   t   popt   sendR   R   t   INDENTt   DEDENTRC   t   objectR   R   R   R   R   R   R   R5   RG   (    (    (    s0   /tmp/pip-build-EndXZ2/flake8/flake8/processor.pyt   <module>   s6   ( W					
		