[HC Offical] SpiderPy - Regular Expression Crawler

Code:
#!/usr/bin/env python
import urllib2, re, sys, argparse, signal
from time import sleep
signal.signal(signal.SIGINT, lambda x,y: sys.exit(0))

print "\033[95m  /\\  /\\/ __\\"
print " / /_/ / /   SpiderPy - A python REGEX crawler"
print "/ __  / /___ Made for http://hackcommunity.com by H3R0"
print "\\/ /_/\\____/ \033[00m"

class Spider():
  def __init__(self, host, args):
    self.host_url =  'http://' + host
    self.links = ['http://' + host + '/']
    self.regex = {
      'EMAIL' : '[a-zA-Z0-9_\.-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-\.]+',
      'IP' : '(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)',
      'PHONE' : '[1|\(]?[2-9]\d{2}\)?[-|\.| ]?\d{3}[-|\.| ]?\d{4}',
    }
    self.wait = 500
    self.method = 'EMAIL'
    self._set_config()
    self._set_args(args)
    self.crawl()

  def _set_config(self):
    try:
      fn = 'spider.conf'
      f = open(fn, 'r')
      c = f.read()
      v = eval(c)
      self.regex = dict(self.regex.items() + v.items())
      f.close()
    except:
      print "\033[93mNo config file found.\nPlease create %s to create more predefined patterns.\033[0m" % fn

  def _set_args(self, args):
    if args.patterns:
      print '\033[94mList of REGEX Patterns to look for on each page...\033[00m'
      for pattern in self.regex:
        print pattern
      sys.exit('')
    if args.timer:
      wait = int(args.timer)
    if args.defined:
      try:
        self.regex[args.defined.upper()]
      except:
        sys.exit('\033[91mNo Such Pattern\033[0m')
      else:
        self.method = args.defined.upper()
    if args.regex:
      self.regex['CUSTOM_CLI_REGEX'] = args.regex
      self.method = 'CUSTOM_CLI_REGEX'

  def _find_links(self, html):
    ls = re.findall('<a[^>]*href="([^"]*)"', html)
    for l in ls:
      i = ''
      if not l.startswith('http') and not l.startswith('//') and not l.startswith('javascript:'):
        i = self.host_url + l if l.startswith('/') else self.host_url + '/' + l
      elif host in l:
        i = l
      if i and not i in self.links:
        self.links.append(i)

  def _output_find(self, url, results):
    print "\033[92m%s" % url
    print results
    print "\033[0m"

  def _run_regex(self, html, url):
    results = re.findall(self.regex[self.method.upper()], html)
    if len(results):
      self._output_find(url, results)

  def crawl(self, n = 0): # Spiders going to spide.
    url = self.links[n]
    try:
      w = urllib2.urlopen(url)
      html = w.read()
      info = w.info().getheader('Content-Type')
      if info.startswith('text') or 'xml' in info:
        self._find_links(html)
        self._run_regex(html, url)
      else:
        print "\033[93mSkipping file type %s\033[0m" % info
    except urllib2.HTTPError:
      #print url
      print '\033[93mThere was an issue with the url, skipping page.\033[0m'
    if n < len(self.links) - 1:
      sleep(self.wait/1000)
      self.crawl(n + 1)

if __name__ == "__main__":
  parser = argparse.ArgumentParser(prog='SpiderPy')
  parser.add_argument('-d', '--defined', default='EMAIL', help='Use a preexisting pattern.  Use -p for patterns.')
  parser.add_argument('-r', '--regex', help='Use a custom regular expression pattern.')
  parser.add_argument('-t', '--timer', help='Time to wait between loading webpages (in milliseconds).')
  parser.add_argument('-p', '--patterns', help='List out all defined patterns.', action='store_true')
  parser.add_argument('host', help='The host name of the website to scrap. (e.g. google.com)')
  args = parser.parse_args()
  host = args.host
  spider = Spider(host, args)