grthtrhthjhtyjytjytkergtrhtrjytjerhrfh4:24 29/09/2026  fc@sdZdZdZddlmZddlmZmZddl Z ddl m Z ddl m Z ddlZddlZdd lTejd ejZejd ejZddlZejd d dddddgZ[eZd eed96e?d:6e@d;6eAd<6eBd=6eCd>6eDd?6eEd@6ZFGdAdde jGddBZHdCdDZIdEdFZJdGdHZKdIZLdJZMeLeJdKeLeKeMZNdLZOdMZPdNZQdOZRdPZSeIePeQeReSZTdQZUeIdRdSeKeUZVdTeUZWeIeVeWZXeIdUeXdVZYeIeYeXeTZZdWZ[dXZ\dYZ]dZZ^d[Z_eIe[d\e[d]Z`eIe[d^e[d_ZaeId`dadbd+dcddded.ZbdfZceIdgdhdiZdeIebecedZeeIeZeeeaeOZfeNefZgeIe[djeIdkdKe[dleIdmdKZheIdneMe`ZieLeIeieZeeeheOZjdodpZki(e\dk6e]dm6e^d\6e_d]6e^dq6e_dr6e^ds6e_dt6e^du6e_dv6e^dw6e_dx6e^dy6e_dz6e^d{6e_d|6e^d}6e_d~6e^d6e_d6e^d6e_d6e^d6e_d6e^d6e_d6e^d6e_d6e^d6e_d6e^du6e_dv6e^d6e_d6dd6dd6dd6dd6dd6dd6ZliZmxdD]ZnenemenzpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord)open)lookupBOM_UTF8N) TextIOWrapper)chain)*z%^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)COMMENTtokenizedetect_encodingNL untokenizeENCODING TokenInfo()[]:,;+-r/|&<>=.%{}z==z!=z<=z>=~^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=@c@s.eZdZddZeddZdS)rcCs.d|jt|jf}d|jd|S)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)type)r(tok_name_replace)selfannotated_typer--/opt/alt/python34/lib64/python3.4/tokenize.py__repr__bszTokenInfo.__repr__cCs4|jtkr)|jtkr)t|jS|jSdS)N)r(OPstringEXACT_TOKEN_TYPES)r+r-r-r. exact_typegs zTokenInfo.exact_typeN)__name__ __module__ __qualname__r/propertyr3r-r-r-r.ras  ztype string start end linecGsddj|dS)Nrrr)join)choicesr-r-r.groupnsr:cGst|dS)Nr)r:)r9r-r-r.anyosr;cGst|dS)N?)r:)r9r-r-r.maybepsr=z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX][0-9a-fA-F]+z 0[bB][01]+z 0[oO][0-7]+z(?:0+|[1-9][0-9]*)z[eE][-+]?[0-9]+z[0-9]+\.[0-9]*z\.[0-9]+z[0-9]+z [0-9]+[jJ]z[jJ]z(?:[bB][rR]?|[rR][bB]?|[uU])?z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'''"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&|^=<>]=?z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z \\\r?\n|\ZcCstj|tjS)N)recompileUNICODE)exprr-r-r._compilesrFr'''r"""b'''b"""R'''R"""B'''B"""br'''br"""bR'''bR"""Br'''Br"""BR'''BR"""rb'''rb"""Rb'''Rb"""rB'''rB"""RB'''RB"""u'''u"""U'''U"""rRbBuUr'r"R'R"b'b"B'B"br'br"Br'Br"bR'bR"BR'BR"rb'rb"rB'rB"Rb'Rb"RB'RB"u'u"U'U"c@seZdZdS) TokenErrorN)r4r5r6r-r-r-r.rs rc@seZdZdS)StopTokenizingN)r4r5r6r-r-r-r.rs rc@s@eZdZddZddZddZddZd S) UntokenizercCs(g|_d|_d|_d|_dS)Nrr)tokensprev_rowprev_colencoding)r+r-r-r.__init__s   zUntokenizer.__init__cCs|\}}||jks9||jkr`||jkr`tdj|||j|jn||j}|r|jjd|d|_n||j}|r|jjd|ndS)Nz+start ({},{}) precedes previous end ({},{})z\ r )rr ValueErrorformatrappend)r+startrowcol row_offset col_offsetr-r-r.add_whitespaces -    zUntokenizer.add_whitespacec Cst|}g}d}x|D]}t|dkrK|j||Pn|\}}}} } |tkr{||_qn|tkrPn|tkr|j|qn|tkr|j | \|_ |_ qnr|t t fkrd}nW|rJ|rJ|d} |dt| krA|jj| t| |_ nd}n|j||jj|| \|_ |_ |t t fkr|j d7_ d|_ qqWdj|jS)NFrTrr)iterlencompatr r ENDMARKERINDENTrDEDENTpoprrNEWLINEr rrr8) r+iterableitindents startlinettok_typetokenrendlineindentr-r-r.r sF              zUntokenizer.untokenizec CsPg}|jj}|dttfk}d}xt|g|D]}|dd\}} |tkrx| |_qAn|ttfkr| d7} n|t kr|rd| } nd}nd}|t kr|j| qAnZ|t kr|j qAn>|ttfkrd}n#|r>|r>||dd}n|| qAWdS)NrFrrTrr) rrrr rr rNAMENUMBERSTRINGrrr) r+rrr toks_appendr prevstringtoktoknumtokvalr-r-r.rs8              zUntokenizer.compatN)r4r5r6rrr rr-r-r-r.rs   %rcCs@t}|j|}|jdk r<|j|j}n|S)aTransform tokens back into Python source code. It returns a bytes object, encoded using the ENCODING token, which is the first token sequence output by tokenize. Each element returned by the iterable must be a token sequence with at least two elements, a token number and token value. If only two tokens are passed, the resulting output is poor. Round-trip invariant for full input: Untokenized source will match input source exactly Round-trip invariant for limited intput: # Output bytes will tokenize the back to the input t1 = [tok[:2] for tok in tokenize(f.readline)] newcode = untokenize(t1) readline = BytesIO(newcode).readline t2 = [tok[:2] for tok in tokenize(readline)] assert t1 == t2 N)rr rencode)rutoutr-r-r.r ?s  cCsd|ddjjdd}|dks=|jdrAdS|d ks\|jdr`dS|S)z(Imitates get_normal_name in tokenizer.c.N _rzutf-8zutf-8-latin-1 iso-8859-1 iso-latin-1latin-1- iso-8859-1- iso-latin-1-)rrr)rrr)lowerreplace startswith)orig_encencr-r-r._get_normal_nameZs" rc s4yjjWntk r*dYnXdd}d}fdd}fdd}|}|jtrd|d d}d }n|s|gfS||}|r||gfStj|s||gfS|}|s||gfS||}|r$|||gfS|||gfS) a The detect_encoding() function is used to detect the encoding that should be used to decode a Python source file. It requires one argument, readline, in the same way as the tokenize() generator. It will call readline a maximum of twice, and return the encoding used (as a string) and a list of any lines (left as bytes) it has read in. It detects the encoding from the presence of a utf-8 bom or an encoding cookie as specified in pep-0263. If both a bom and a cookie are present, but disagree, a SyntaxError will be raised. If the encoding cookie is an invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, 'utf-8-sig' is returned. If no encoding is specified, then the default of 'utf-8' will be returned. NFzutf-8c s(y SWntk r#dSYnXdS)N) StopIterationr-)readliner-r. read_or_stop}s  z%detect_encoding..read_or_stopcs;y|jd}WnEtk rZd}dk rJdj|}nt|YnXtj|}|stdSt|jd}yt|}WnIt k rdkrd|}ndj|}t|YnXr7|dkr*dkr d}ndj}t|n|d 7}n|S) Nzutf-8z'invalid or missing encoding declarationz {} for {!r}rzunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig) decodeUnicodeDecodeErrorr SyntaxError cookie_rematchrr:r LookupError)r line_stringmsgrrcodec) bom_foundfilenamer-r. find_cookies6           z$detect_encoding..find_cookieTrz utf-8-sig)__self__nameAttributeErrorrrblank_rer)rrdefaultrrfirstsecondr-)rrrr.r es8  &         c Csst|d}yHt|j\}}|jdt||dd}d|_|SWn|jYnXdS)zXOpen a file in read only mode using the encoding detected by detect_encoding(). rbrline_bufferingTrcN) _builtin_openr rseekrmodeclose)rbufferrlinestextr-r-r.rs   rcCs_ddlm}m}t|\}}t|d}|d}t||||j|S)a The tokenize() generator requires one argment, readline, which must be a callable object which provides the same interface as the readline() method of built-in file objects. Each call to the function should return one line of input as bytes. Alternately, readline can be a callable function terminating with StopIteration: readline = open(myfile, 'rb').__next__ # Example of alternate readline The generator produces 5-tuples with these members: the token type; the token string; a 2-tuple (srow, scol) of ints specifying the row and column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed is the logical line; continuation lines are included. The first token sequence will always be an ENCODING token which tells you which encoding was used to decode the bytes stream. r)rrepeatr) itertoolsrrr r _tokenize__next__)rrrrconsumedrl_genemptyr-r-r.r s  ccs d}}}d}d\}}d}dg} |dk rj|dkrPd}ntt|dddVnxNy |} Wntk rd} YnX|dk r| j|} n|d7}dt| } } |r| std| n|j| }|rZ|jd} }tt|| d|| ||f|| Vd\}}d}ql|r| d dd kr| d!dd krtt || | |t| f|Vd}d}qmql|| }|| }qmn|dkrH| rH| sPnd}xv| | krz| | d kr.|d7}n?| | dkrS|t dt }n| | dkrld}nP| d7} qW| | krPn| | dkr| | dkr7| | dj d}| t|}tt ||| f|| t|f| Vtt | |d||f|t| f| Vqmtt t f| | dk| | d|| f|t| f| Vqmn|| d"kr| j|tt| d| |df|| f| Vnx|| d#krD|| krtdd|| | fn| dd$} ttd|| f|| f| VqWn$| sftd|dfnd}xH| | krttj| | }|r~|jd\}}||f||f|}}} ||krqon| ||| |}}||ks+|dkrE|dkrE|dkrEtt|||| Vq|dkr}t|dkrft nt|||| Vq|dkrtt |||| Vq|tkr<tt|}|j| | }|r|jd} | || }tt|||| f| Vq{||f} | |d}| }Pq|tkst|dd tkst|dd tkr|d%dkr||f} tt|pt|dpt|d }| |dd}}| }Pq{tt|||| Vq|jrtt|||| Vq|dkr2d}q|dkrK|d7}n|dkrd|d8}ntt|||| Vqott | | || f|| df| V| d7} qoWqmWx;| ddD])}ttd|df|dfdVqWttd|df|dfdVdS)&Nr 0123456789rz utf-8-sigzutf-8rrzEOF in multi-line stringrz\ rz\ r  z# #z z3unindent does not match any outer indentation levelz zEOF in multi-line statementr!z... \z([{z)]})rr)rr)rr)rrrrrr)rr rrrrrrr ERRORTOKENtabsizerstriprr rrIndentationErrorrrF PseudoTokenspanrr triple_quotedendpats single_quoted isidentifierrr0r)rrlnumparenlev continuednumcharscontstrneedcontcontlinerrposmaxstrstartendprogendmatchrcolumn comment_tokennl_pos pseudomatchrsposeposrinitialrr-r-r.rs            2       $#'  0 *   $                'rcCs t|dS)N)r)rr-r-r.generate_tokenssrc sddl}ddddfdd}|jdd}|jdd d d d d dd|jdddddddd|j}y|jr|j}t|d}tt|j}WdQXnd}t t j jd}x^|D]V}|j }|j r|j}nd|j|j} td| t||jfqWWnbtk r} z?| jddd\} } || jd|| | fWYdd} ~ Xntk r } z5| jd\} } || jd|| | fWYdd} ~ Xntk r:} z|| |WYdd} ~ Xntk rh} z|| WYdd} ~ XnQtk rtdYn6tk r} zd| WYdd} ~ XnXdS)NrcSst|dtjdS)Nfile)printsysstderr)messager-r-r.perrorszmain..perrorcsg|r+|f||f}d|n+|rHd||fnd|tjddS)Nz%s:%d:%d: error: %sz %s: error: %sz error: %sr)rexit)rrlocationargs)rr-r.errorszmain..errorprogzpython -m tokenizedestrnargsr<metavarz filename.pyhelpz'the file to tokenize; defaults to stdinz-ez--exactexactaction store_truez(display token names using the exact typerzz %d,%d-%d,%d:z%-20s%-15s%-15rrrz interrupted zunexpected error: %s)argparseArgumentParser add_argument parse_argsrrlistr rrrstdinr(r!r3rrrr)r1rrrrOSErrorKeyboardInterrupt Exception) r$rparserrrfrr token_type token_rangeerrrr r-)rr.mainsN           // r2__main__)r>r?rGrHrKrLrIrJrMrNrOrPrSrTrQrRrUrVrWrXr[r\rYrZr]r^r_r`rarb)r@rArirjrkrlrmrnrorprqrrrsrtrurvrwrxryrzr{r|r}r~rrrrrr)}__doc__ __author__ __credits__builtinsrrcodecsrr collectionsiorrrrBrrrCASCIIrr__all__N_TOKENSrr)r r LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE CIRCUMFLEX LEFTSHIFT RIGHTSHIFT DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL SLASHEQUAL PERCENTEQUAL AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUALATr2 namedtuplerr:r;r= WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent PointfloatExpfloat Floatnumber ImagnumberNumber StringPrefixSingleDoubleSingle3Double3TripleStringOperatorBracketSpecialFunny PlainTokenTokenContStr PseudoExtrasrrFrrrrrr,rrrr rr r rrr2r4r-r-r-r.sF            "             _  ]    <