Download System using Requests

Inori · 12-16-2015, 02:34 PM

Using the super popular python module requests, which makes http, POST, and RESTful requests and traversal humanly possible (screw you, urllib2), you can 'get' a raw file from the url source and download it chunk by chunk as you would with raw html to use requests normally.

This project is fairly bug-free, save for the download bar not completing on some operations, but other than that, it's fully functional and works pretty well.

(Full code on github)

Spoiler: source code
__main__.py (used to execute code from the parent directory):
Code:
from internals import *

cmds=['e','q','r','v']
q=[]

def main():
	a=raw_input('~> ').split()
	if a[0] in cmds:
		if a[0]=='e':
			exit()
		elif a[0]=='q':
			if len(a)<2:
				print '!> No file supplied'
			else:
				uri=a[1]
				x=requests.get(uri)
				name=(a[1],uri.split('/')[-1])[len(a)<2]
				i=0
				if x.status_code==200:
					for c in x: i+=1
					q.append((name,uri.split('/')[2],requests.get(uri,stream=True),i,a))
				else:
					print'#> %s returned status code %s'%(uri.split('/')[2],r.status_code)
		elif a[0]=='r':
			while len(q)>0:
				runq(q)
			print '\n#> download(s) complete.'
		elif a[0]=='v':
			for i in range(len(q)):
				print '#%s: %s,%s'%(i+1,q[i][-1][1],q[i][-1][2])

	else:
		print '!> %s is not a valid command'%a[0]
	main()

if tests():
	main()


internals.py:
Code:
import requests
from sys import stdout
from os import remove,makedirs,path
from threading import Thread

def tests():
	if not path.exists('./PyDownload/Downloads'):
		makedirs('./PyDownload/Downloads')
	try:
		requests.get('https://google.com').status_code
		return True
	except requests.exceptions.ConnectionError:
		print '#> Currently disconnected, try again later.'
		return False

def prog(p):
    block=int(round(30*p))
    text="\r#> [{0}] {1}% ".format("*"*block+" "*(30-block),str(p*100)[:5])
    stdout.write(text)
    stdout.flush()

def download(u,s,r,i,a,n=0):
	try:
		open(u,'r').close()
		print '!> file %s already exists'%u
	except IOError:
		try:
			print '#> downloading file %s from %s'%(u.split('/')[-1],s)
			with open('./PyDownload/Downloads/%s'%((u.split('/')[-1])+'.html'*('.' not in u.split('/')[-1]),a[-1])[len(a)>2],'wb') as f:
				for c in r.iter_content(chunk_size=512):
					n+=1
					f.write(c)
					prog(n/float(i))
		except KeyboardInterrupt:
			print '\n!> download canceled.'
			remove(u)
			exit()

def runq(q):
	for i in q:
		worker=Thread(target=download,args=i)
		worker.start()
		worker.join()
		q.remove(i)

Download System using Requests
Author Message