+- Sinisterly (https://sinister.li)
+-- Forum: Coding (https://sinister.li/Forum-Coding)
+--- Forum: Python (https://sinister.li/Forum-Python)
+--- Thread: Download System using Requests (/Thread-Download-System-using-Requests)
Download System using Requests - Inori - 12-16-2015
Using the super popular python module requests, which makes http, POST, and RESTful requests and traversal humanly possible (screw you, urllib2), you can 'get' a raw file from the url source and download it chunk by chunk as you would with raw html to use requests normally.
This project is fairly bug-free, save for the download bar not completing on some operations, but other than that, it's fully functional and works pretty well.
(Full code on github)
Spoiler: source code
__main__.py (used to execute code from the parent directory):
internals.py:
Code:
from internals import *
cmds=['e','q','r','v']
q=[]
def main():
a=raw_input('~> ').split()
if a[0] in cmds:
if a[0]=='e':
exit()
elif a[0]=='q':
if len(a)<2:
print '!> No file supplied'
else:
uri=a[1]
x=requests.get(uri)
name=(a[1],uri.split('/')[-1])[len(a)<2]
i=0
if x.status_code==200:
for c in x: i+=1
q.append((name,uri.split('/')[2],requests.get(uri,stream=True),i,a))
else:
print'#> %s returned status code %s'%(uri.split('/')[2],r.status_code)
elif a[0]=='r':
while len(q)>0:
runq(q)
print '\n#> download(s) complete.'
elif a[0]=='v':
for i in range(len(q)):
print '#%s: %s,%s'%(i+1,q[i][-1][1],q[i][-1][2])
else:
print '!> %s is not a valid command'%a[0]
main()
if tests():
main()internals.py:
Code:
import requests
from sys import stdout
from os import remove,makedirs,path
from threading import Thread
def tests():
if not path.exists('./PyDownload/Downloads'):
makedirs('./PyDownload/Downloads')
try:
requests.get('https://google.com').status_code
return True
except requests.exceptions.ConnectionError:
print '#> Currently disconnected, try again later.'
return False
def prog(p):
block=int(round(30*p))
text="\r#> [{0}] {1}% ".format("*"*block+" "*(30-block),str(p*100)[:5])
stdout.write(text)
stdout.flush()
def download(u,s,r,i,a,n=0):
try:
open(u,'r').close()
print '!> file %s already exists'%u
except IOError:
try:
print '#> downloading file %s from %s'%(u.split('/')[-1],s)
with open('./PyDownload/Downloads/%s'%((u.split('/')[-1])+'.html'*('.' not in u.split('/')[-1]),a[-1])[len(a)>2],'wb') as f:
for c in r.iter_content(chunk_size=512):
n+=1
f.write(c)
prog(n/float(i))
except KeyboardInterrupt:
print '\n!> download canceled.'
remove(u)
exit()
def runq(q):
for i in q:
worker=Thread(target=download,args=i)
worker.start()
worker.join()
q.remove(i)