Vega strike Python Modules doc: RobotFileParser Class Reference

All Data Structures Namespaces Files Functions Variables

Public Member Functions
def	__init__

def	mtime

def	modified

def	set_url

def	read

def	parse

def	can_fetch

def	__str__

Data Fields
	entries

	disallow_all

	allow_all

	last_checked

	url

	path

	errcode

Detailed Description

Definition at line 22 of file robotparser.py.

Constructor & Destructor Documentation

def __init__	(	self,
		url = `''`
	)

Definition at line 23 of file robotparser.py.

 
     def __init__(self, url=''):
         self.entries = []
         self.disallow_all = 0
         self.allow_all = 0
         self.set_url(url)
         self.last_checked = 0

Member Function Documentation

def __str__ ( self)

Definition at line 139 of file robotparser.py.

References RobotFileParser.entries, and locale.str().

 
     def __str__(self):
         ret = ""
         for entry in self.entries:
             ret = ret + str(entry) + "\n"
         return ret
 

def can_fetch	(	self,
		useragent,
		url
	)

using the parsed robots.txt decide if useragent can fetch url

Definition at line 121 of file robotparser.py.

References RobotFileParser.allow_all, RobotFileParser.disallow_all, RobotFileParser.entries, urllib.quote(), and urlparse.urlparse().

 
     def can_fetch(self, useragent, url):
         """using the parsed robots.txt decide if useragent can fetch url"""
         _debug("Checking robot.txt allowance for:\n  user agent: %s\n  url: %s" %
                (useragent, url))
         if self.disallow_all:
             return 0
         if self.allow_all:
             return 1
         # search for given user agent matches
         # the first match counts
         url = urllib.quote(urlparse.urlparse(url)[2]) or "/"
         for entry in self.entries:
             if entry.applies_to(useragent):
                 return entry.allowance(url)
         # agent not found ==> access granted
         return 1
 

def modified ( self)

Definition at line 33 of file robotparser.py.

References RobotFileParser.last_checked.

 
     def modified(self):
         import time
         self.last_checked = time.time()

def mtime ( self)

Definition at line 30 of file robotparser.py.

References RobotFileParser.last_checked.

 
     def mtime(self):
         return self.last_checked

def parse	(	self,
		lines
	)

parse the input lines from a robot.txt file.
   We allow that a user-agent: line is not preceded by
   one or more blank lines.

Definition at line 56 of file robotparser.py.

References string.lower(), locale.str(), and string.strip().

 
     def parse(self, lines):
         """parse the input lines from a robot.txt file.
            We allow that a user-agent: line is not preceded by
            one or more blank lines."""
         state = 0
         linenumber = 0
         entry = Entry()
 
         for line in lines:
             line = line.strip()
             linenumber = linenumber + 1
             if not line:
                 if state==1:
                     _debug("line %d: warning: you should insert"
                            " allow: or disallow: directives below any"
                            " user-agent: line" % linenumber)
                     entry = Entry()
                     state = 0
                 elif state==2:
                     self.entries.append(entry)
                     entry = Entry()
                     state = 0
             # remove optional comment and strip line
             i = line.find('#')
             if i>=0:
                 line = line[:i]
             line = line.strip()
             if not line:
                 continue
             line = line.split(':', 1)
             if len(line) == 2:
                 line[0] = line[0].strip().lower()
                 line[1] = line[1].strip()
                 if line[0] == "user-agent":
                     if state==2:
                         _debug("line %d: warning: you should insert a blank"
                                " line before any user-agent"
                                " directive" % linenumber)
                         self.entries.append(entry)
                         entry = Entry()
                     entry.useragents.append(line[1])
                     state = 1
                 elif line[0] == "disallow":
                     if state==0:
                         _debug("line %d: error: you must insert a user-agent:"
                                " directive before this line" % linenumber)
                     else:
                         entry.rulelines.append(RuleLine(line[1], 0))
                         state = 2
                 elif line[0] == "allow":
                     if state==0:
                         _debug("line %d: error: you must insert a user-agent:"
                                " directive before this line" % linenumber)
                     else:
                         entry.rulelines.append(RuleLine(line[1], 1))
                 else:
                     _debug("line %d: warning: unknown key %s" % (linenumber,
                                line[0]))
             else:
                 _debug("line %d: error: malformed line %s"%(linenumber, line))
         if state==2:
             self.entries.append(entry)
         _debug("Parsed rules:\n%s" % str(self))
 

def read ( self)

Definition at line 41 of file robotparser.py.

References RobotFileParser.url.

 
     def read(self):
         opener = URLopener()
         f = opener.open(self.url)
         lines = f.readlines()
         self.errcode = opener.errcode
         if self.errcode == 401 or self.errcode == 403:
             self.disallow_all = 1
             _debug("disallow all")
         elif self.errcode >= 400:
             self.allow_all = 1
             _debug("allow all")
         elif self.errcode == 200 and lines:
             _debug("parse lines")
             self.parse(lines)

Public Member Functions

Data Fields

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Field Documentation