CareerCup

Hi, I'm not sure if this is Exactly the answer to the problem you posted, but this is would be my approach to the problem.
Any inputs or suggestions are most welcome!

import lxml
from lxml import etree
import urllib2
class webCrawler(object):
	self.urlStack = set() #Set to avoid duplicates in the list
	def __init__(self,baseURL):
		"""The init function(can do database connection setup here)"""
		self.urlStack.add(baseURL)

	def checkURL(self,url):
		"""Check if the url already exists in the database and return True or False"""
	def db_call(self,url,html):
		"""Insert into the db/disk the URL and the html content of the URL"""
	
	def crawl(self,url):
		try:
			html = urllib2.urlopen(url).read()
		except:
			print "Unable to retrieve HTML"
			if len(self.urlStack)>0:
				self.crawl(self.urlStack.pop())
		href_xpath = "//a/@href"
		self.db_call(url,html)
		tree = lxml.etree.HTML(html)
		hrefs = tree.xpath(href_xpath)
		for each in hrefs:
			if each.find("amazon.com")<=0 or (self.checkURL(url)):
				hrefs.remove(each)
		for each in hrefs:
			self.urlStack.add(each)
		if len(self.urlStack)>0:
			self.crawl(self.urlStack.pop())

This is mine. Has depth limit so that it won't crawl all of Amazon's pages.

from collections import deque
import urllib2
import re

class crawler: 
    def __init__(self):
        self.visited = set()

    def crawl(self, url, max_depth):
        queue = deque()
        queue.append((url, 0))

        while queue:
            url, depth = queue.popleft()
            if depth < max_depth:
                try:
                    source = urllib2.urlopen(url).read()
                except:
                    continue
                print url
                href_pos = [m.end() for m in re.finditer(r'href=', source)]
                for pos in href_pos:
                    quote = source[pos]
                    end_pos = source.find(quote, pos + 1)
                    href = source[pos + 1 : end_pos]
                    if href not in self.visited and href.find('amazon.com') >= 0:
                        self.visited.add(href)
                        queue.append((href, depth + 1))
        
        return self.visited