[HC Offical] SpiderPy - Regular Expression Crawler 07-25-2014, 12:22 AM
#1
Hello all, I decided to write another tool for HC! This time I came up with SpiderPy. It's a crawler that, when given a target, recursively searches through the website and runs a regex against every page.
Feature List:
- Crawls Websites based on open links
- Can run predefined REGEX (Email, Phone, IP address)
- Can run user defined regex
- Can set time delay between page scraps
- Can create config file to add additional predefined regex
Usage Examples:
Source:
Screen Shot:
![[Image: xODNQtY.jpg]](http://i.imgur.com/xODNQtY.jpg)
Feature List:
- Crawls Websites based on open links
- Can run predefined REGEX (Email, Phone, IP address)
- Can run user defined regex
- Can set time delay between page scraps
- Can create config file to add additional predefined regex
Usage Examples:
Code:
# Scan example.com for emails with a 1 sec delay
./spiderpy -d EMAIL -t 1000 example.com
# Scan example.com for custom regex
./spiderpy -r '".*\.pdf' example.com
# Show all predefined(and user defined) regex
./spiderpy -pSource:
Code:
#!/usr/bin/env python
import urllib2, re, sys, argparse, signal
from time import sleep
signal.signal(signal.SIGINT, lambda x,y: sys.exit(0))
print "\033[95m /\\ /\\/ __\\"
print " / /_/ / / SpiderPy - A python REGEX crawler"
print "/ __ / /___ Made for http://hackcommunity.com by H3R0"
print "\\/ /_/\\____/ \033[00m"
class Spider():
def __init__(self, host, args):
self.host_url = 'http://' + host
self.links = ['http://' + host + '/']
self.regex = {
'EMAIL' : '[a-zA-Z0-9_\.-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-\.]+',
'IP' : '(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)',
'PHONE' : '[1|\(]?[2-9]\d{2}\)?[-|\.| ]?\d{3}[-|\.| ]?\d{4}',
}
self.wait = 500
self.method = 'EMAIL'
self._set_config()
self._set_args(args)
self.crawl()
def _set_config(self):
try:
fn = 'spider.conf'
f = open(fn, 'r')
c = f.read()
v = eval(c)
self.regex = dict(self.regex.items() + v.items())
f.close()
except:
print "\033[93mNo config file found.\nPlease create %s to create more predefined patterns.\033[0m" % fn
def _set_args(self, args):
if args.patterns:
print '\033[94mList of REGEX Patterns to look for on each page...\033[00m'
for pattern in self.regex:
print pattern
sys.exit('')
if args.timer:
wait = int(args.timer)
if args.defined:
try:
self.regex[args.defined.upper()]
except:
sys.exit('\033[91mNo Such Pattern\033[0m')
else:
self.method = args.defined.upper()
if args.regex:
self.regex['CUSTOM_CLI_REGEX'] = args.regex
self.method = 'CUSTOM_CLI_REGEX'
def _find_links(self, html):
ls = re.findall('<a[^>]*href="([^"]*)"', html)
for l in ls:
i = ''
if not l.startswith('http') and not l.startswith('//') and not l.startswith('javascript:'):
i = self.host_url + l if l.startswith('/') else self.host_url + '/' + l
elif host in l:
i = l
if i and not i in self.links:
self.links.append(i)
def _output_find(self, url, results):
print "\033[92m%s" % url
print results
print "\033[0m"
def _run_regex(self, html, url):
results = re.findall(self.regex[self.method.upper()], html)
if len(results):
self._output_find(url, results)
def crawl(self, n = 0): # Spiders going to spide.
url = self.links[n]
try:
w = urllib2.urlopen(url)
html = w.read()
info = w.info().getheader('Content-Type')
if info.startswith('text') or 'xml' in info:
self._find_links(html)
self._run_regex(html, url)
else:
print "\033[93mSkipping file type %s\033[0m" % info
except urllib2.HTTPError:
#print url
print '\033[93mThere was an issue with the url, skipping page.\033[0m'
if n < len(self.links) - 1:
sleep(self.wait/1000)
self.crawl(n + 1)
if __name__ == "__main__":
parser = argparse.ArgumentParser(prog='SpiderPy')
parser.add_argument('-d', '--defined', default='EMAIL', help='Use a preexisting pattern. Use -p for patterns.')
parser.add_argument('-r', '--regex', help='Use a custom regular expression pattern.')
parser.add_argument('-t', '--timer', help='Time to wait between loading webpages (in milliseconds).')
parser.add_argument('-p', '--patterns', help='List out all defined patterns.', action='store_true')
parser.add_argument('host', help='The host name of the website to scrap. (e.g. google.com)')
args = parser.parse_args()
host = args.host
spider = Spider(host, args)Screen Shot:
(This post was last modified: 07-29-2014, 09:03 PM by zimba.)
![[Image: iQ3pcQu.png]](http://i.imgur.com/iQ3pcQu.png)
BTC Address: 1DCKgDaWcmc9dxBkhe9qrTQtrQpoFUzXdn
![[+]](https://sinister.li/images/modern/collapse_collapsed.png)
![[Image: wvBFmA5.png]](http://i.imgur.com/wvBFmA5.png)
![[Image: KLGYoXV.png]](http://i.imgur.com/KLGYoXV.png)
![[Image: 5R6Js8r.gif]](http://i.imgur.com/5R6Js8r.gif)
