tokenize.py File Reference

Go to the source code of this file.

Namespaces
	tokenize

Constant Groups
	tokenize

Functions
def	group
def	any
def	maybe

Variables
string	__author__ = 'Ka-Ping Yee <ping@lfw.org>'
	__credits__ = \
list	__all__ = [x for x in dir(token) if x[0] != '_']
	COMMENT = N_TOKENS
int	NL = N_TOKENS+1
string	Whitespace = r'[ \f\t]*'
string	Comment = r'#[^\r\n]*'
tuple	Ignore = Whitespace+any(r'\\\r?\n' + Whitespace)
string	Name = r'[a-zA-Z_]\w*'
string	Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
string	Octnumber = r'0[0-7]*[lL]?'
string	Decnumber = r'[1-9]\d*[lL]?'
tuple	Intnumber = group(Hexnumber, Octnumber, Decnumber)
string	Exponent = r'[eE][-+]?\d+'
tuple	Pointfloat = group(r'\d+\.\d*', r'\.\d+')
string	Expfloat = r'\d+'
tuple	Floatnumber = group(Pointfloat, Expfloat)
tuple	Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
tuple	Number = group(Imagnumber, Floatnumber, Intnumber)
string	Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
string	Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
string	Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
string	Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'