­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ U ,ad@sDdZdZddlmZddlmZmZddlZddl m Z ddl Z ddl Z ddlZddlTddlmZe d e jZe d e jZddlZejd d d ddgZ[GdddeddZddZddZddZdZdZeedeeeZdZdZdZ dZ!dZ"eee e!e"Z#d Z$ed!d"ee$Z%d#e$Z&ee%e&Z'ed$e'd%Z(ee(e'e#Z)d&d'Z*d(d)Z+ee*Z,d*Z-d+Z.d,Z/d-Z0ee,d.e,d/Z1ee,d0e,d1Z2ee3e j4e5ed2d3Z6ed4e6Z7ee)e7e2eZ8ee8Z9ee,d5ed6de,d7ed8dZ:ed9ee1Z;eee;e)e7e:eZe-e=e>d6<e.e=e>d8<e/e=e>d.<e0e=e>d/<q(e?Z@e?ZAe*D]JZBeBd8eBd6fD]ZCe@DeCqeBd/eBd.fD]ZCeADeCqqrd:ZEGd;d<dd>eFZHGd?d@d@ZIdAdZJdBdCZKdDd ZLdEdFZdGd ZMdHdIZNdJd ZOdKdLZPeQdMkr@ePdS)NzKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord)open)lookupBOM_UTF8N) TextIOWrapper)*)EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)tokenizegenerate_tokensdetect_encoding untokenize TokenInfoc@s eZdZddZeddZdS)r cCs$d|jt|jf}d|j|dS)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r tok_name_replace)selfannotated_typer-/opt/alt/python38/lib64/python3.8/tokenize.py__repr__.s zTokenInfo.__repr__cCs(|jtkr|jtkrt|jS|jSdSN)r OPstringrrrrr exact_type3s zTokenInfo.exact_typeN)__name__ __module__ __qualname__rpropertyrrrrrr -sztype string start end linecGsdd|dS)N(|))joinchoicesrrrgroup:r$cGs t|dS)Nrr$r"rrrany;r%r'cGs t|dS)N?r&r"rrrmaybe<r%r)z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]cCs^ddddddg}dh}|D]>}t|D].}tjdd |DD]}|d|q@q(q|S) NbrufbrfrcSsg|]}||fqSr)upper).0crrr ^sz(_all_string_prefixes..) _itertools permutationsproductaddr!)_valid_string_prefixesresultprefixtr,rrr_all_string_prefixesSsr=cCst|tjSr)recompileUNICODE)exprrrr_compilebsrBz[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"T)reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z \\\r?\n|\Zc@s eZdZdS) TokenErrorNrrrrrrrrGsrGc@s eZdZdS)StopTokenizingNrHrrrrrIsrIc@s,eZdZddZddZddZddZd S) UntokenizercCsg|_d|_d|_d|_dS)Nr)tokensprev_rowprev_colencodingrrrr__init__szUntokenizer.__init__cCs|\}}||jks&||jkr>||jkr>td|||j|j||j}|rb|jd|d|_||j}|r|jd|dS)Nz+start ({},{}) precedes previous end ({},{})\ r )rMrN ValueErrorformatrLappend)rstartrowcol row_offset col_offsetrrradd_whitespaces  zUntokenizer.add_whitespacec Cs6t|}g}d}|D]}t|dkr8|||q*|\}}}} } |tkrV||_q|tkrdq*|tkrz||qnl|tkr| | \|_ |_ qnL|t t fkrd}n:|r|r|d} |dt| kr|j| t| |_ d}|||j|| \|_ |_ |t t fkr|j d7_ d|_ qd|jS)NFTrKrr0)iterlencompatENCODINGrO ENDMARKERINDENTrUDEDENTpoprMrNNEWLINENLrLr[r!) riterableitindents startliner<tok_typetokenrVendlineindentrrrr sF            zUntokenizer.untokenizec Csg}|jj}|dttfk}d}t|g|D]}|dd\}} |tkrR| |_q.|tt fkrf| d7} |t kr|rzd| } d}nd}|t kr|| q.n>|t kr| q.n*|ttfkrd}n|r|r||dd}|| q.dS)NrFr\rRTr])rLrUrfrgr5chainrarONAMENUMBERSTRINGrcrdre) rrmrhrj toks_appendrk prevstringtoktoknumtokvalrrrr`s8    zUntokenizer.compatN)rrrrPr[r r`rrrrrJs %rJcCs*t}||}|jdk r&||j}|Sr)rJr rOencode)rhutoutrrrr s    cCsH|dddd}|dks*|dr.dS|dks@|drDdS|S) N _-utf-8zutf-8-)zlatin-1 iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-r)lowerreplace startswith)orig_encencrrr_get_normal_namesrcsz jjWntk r$dYnXdd}d}fdd}fdd}|}|trpd|dd}d }|s||gfS||}|r||gfSt|s||gfS|}|s||gfS||}|r|||gfS|||gfS) NFrcs$zWStk rYdSXdSNr%) StopIterationrreadlinerr read_or_stop?sz%detect_encoding..read_or_stopcsz|d}Wn4tk rBd}dk r6d|}t|YnXt|}|sVdSt|d}z t|}Wn:t k rdkrd|}n d|}t|YnXr|dkr؈dkrd}n d}t||d 7}|S) Nrz'invalid or missing encoding declarationz {} for {!r}rKzunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig) decodeUnicodeDecodeErrorrT SyntaxError cookie_rematchrr$r LookupError)ro line_stringmsgrrOcodec) bom_foundfilenamerr find_cookieEs8     z$detect_encoding..find_cookieT utf-8-sig)__self__nameAttributeErrorrrblank_rer)rrOdefaultrrfirstsecondr)rrrrr 's8   &       cCsXt|d}z2t|j\}}|dt||dd}d|_|WS|YnXdS)NrbrT)line_bufferingr+) _builtin_openr rseekrmodeclose)rbufferrOlinestextrrrrs  rcCs6t|\}}td}t|t|d|}t|j|Sr)r r5repeatrqr^ _tokenize__next__)rrOconsumedemptyrl_genrrrrs  ccsld}}}d}d\}}d}dg} |dk rH|dkr6d}tt|dddVd} d} z| } |} Wntk rvd} YnX|dk r| |} |d 7}dt| } } |rp| std ||| }|r|d} }tt|| d||||f|| Vd\}}d}nf|rZ| d dd krZ| d ddkrZtt || ||t| f|Vd}d}qPn|| }|| }qPn|dkr|s| sqd}| | kr| | dkr|d 7}n8| | dkr|t d t }n| | dkrd}nq| d 7} q| | krq| | dkr| | dkr^| | d d}tt ||| f|| t|f| V| t|7} tt | | d|| f|t| f| VqP|| dkr| |tt| d| |df|| f| V|| dkr.|| krtdd|| | f| dd} ttd|| f|| f| Vqn| s*td|dfd}| | krPtt| | }|r|d \}}||f||f|}}} ||krq.| ||| |}}||ks|dkr|dkr|dkrtt|||| Vq|dkr|dkrtt |||| Vntt|||| Vq|dkr2tt |||| Vq|tkrtt|}|| | }|r|d} | || }tt|||| f| Vn||f}| |d}| }qPq|tks|ddtks|ddtkrF|ddkr2||f}tt|pt|d pt|d}| |dd }}| }qPntt|||| Vnf|rdtt|||| VnH|dkrtd }n8|dkr|d 7}n|d kr|d 8}tt|||| Vn*tt | | || f|| d f| V| d 7} q.qP| r | ddkr ttd|d t| f|d t| d fdV| d dD] }ttd|df|dfdVq,ttd|df|dfdVdS)!Nr 0123456789)r0rrr)rrr0r%rKzEOF in multi-line stringrQz\ rR  z# #z r]z3unindent does not match any outer indentation levelz zEOF in multi-line statement.z...r\r \z([{z)]})r rarrr_rGrrnrt ERRORTOKENtabsizerstripCOMMENTrgrUrcIndentationErrorrdrB PseudoTokenspanrsrf triple_quotedendpats single_quotedget isidentifierrrrrb)rrOlnumparenlev continuednumcharscontstrneedcontcontlinerj last_lineroposmaxstrstartendprogendmatchrncolumn comment_token pseudomatchrVsposeposrminitialrprrrrs<      *         "                         .rcCs t|dSr)rrrrrr dsc s$ddl}dddfdd }|jdd}|jdd d d d |jd ddddd|}z|jr|j}t|d}tt|j}W5QRXnd}t t j jd}|D]>}|j }|j r|j}d|j|j} td| t||jfqWn8tk r6} z0| jddd\} } || jd|| | fW5d} ~ XYntk r|} z(| jd\} } || jd|| | fW5d} ~ XYntk r} z|| |W5d} ~ XYnxtk r} z || W5d} ~ XYnNtk rtdYn2tk r} zd| W5d} ~ XYnXdS)NrcSstj|tjddS)Nr)sysstderrwrite)messagerrrperrorps zmain..perrorcsR|r"|f||f}d|n"|r8d||fn d|tddS)Nz%s:%d:%d: error: %sz %s: error: %sz error: %srK)rexit)rrlocationargsrrrerrorts zmain..errorzpython -m tokenize)progrr(z filename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact store_truez(display token names using the exact type)ractionrrzz %d,%d-%d,%d:z%-20s%-15s%-15rrKrz interrupted zunexpected error: %s)NN)argparseArgumentParser add_argument parse_argsrrlistrrrrstdinr rrrVrnprintrrrrrGrOSErrorKeyboardInterrupt Exception) rrparserrrr-rLrm token_type token_rangeerrrorrrrmainlsT   &&  r__main__)R __author__ __credits__builtinsrrcodecsrr collectionsior itertoolsr5r>rrmrr?ASCIIrr__all__ namedtupler r$r'r) WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent PointfloatExpfloat Floatnumber ImagnumberNumberr=rB StringPrefixSingleDoubleSingle3Double3TripleStringmapescapesortedSpecialFunny PlainTokenTokenContStr PseudoExtrasrr_prefixsetrrr<r,r8rrrGrIrJr rr rrr rrrrrrs               _ ]8=