Sinisterly

Full Version: Download System using Requests
You're currently viewing a stripped down version of our content. View the full version with proper formatting.
Using the super popular python module requests, which makes http, POST, and RESTful requests and traversal humanly possible (screw you, urllib2), you can 'get' a raw file from the url source and download it chunk by chunk as you would with raw html to use requests normally.

This project is fairly bug-free, save for the download bar not completing on some operations, but other than that, it's fully functional and works pretty well.

(Full code on github)

Spoiler: source code
__main__.py (used to execute code from the parent directory):
Code:
from internals import *

cmds=['e','q','r','v']
q=[]

def main():
    a=raw_input('~> ').split()
    if a[0] in cmds:
        if a[0]=='e':
            exit()
        elif a[0]=='q':
            if len(a)<2:
                print '!> No file supplied'
            else:
                uri=a[1]
                x=requests.get(uri)
                name=(a[1],uri.split('/')[-1])[len(a)<2]
                i=0
                if x.status_code==200:
                    for c in x: i+=1
                    q.append((name,uri.split('/')[2],requests.get(uri,stream=True),i,a))
                else:
                    print'#> %s returned status code %s'%(uri.split('/')[2],r.status_code)
        elif a[0]=='r':
            while len(q)>0:
                runq(q)
            print '\n#> download(s) complete.'
        elif a[0]=='v':
            for i in range(len(q)):
                print '#%s: %s,%s'%(i+1,q[i][-1][1],q[i][-1][2])

    else:
        print '!> %s is not a valid command'%a[0]
    main()

if tests():
    main()

internals.py:
Code:
import requests
from sys import stdout
from os import remove,makedirs,path
from threading import Thread

def tests():
    if not path.exists('./PyDownload/Downloads'):
        makedirs('./PyDownload/Downloads')
    try:
        requests.get('https://google.com').status_code
        return True
    except requests.exceptions.ConnectionError:
        print '#> Currently disconnected, try again later.'
        return False

def prog(p):
    block=int(round(30*p))
    text="\r#> [{0}] {1}% ".format("*"*block+" "*(30-block),str(p*100)[:5])
    stdout.write(text)
    stdout.flush()

def download(u,s,r,i,a,n=0):
    try:
        open(u,'r').close()
        print '!> file %s already exists'%u
    except IOError:
        try:
            print '#> downloading file %s from %s'%(u.split('/')[-1],s)
            with open('./PyDownload/Downloads/%s'%((u.split('/')[-1])+'.html'*('.' not in u.split('/')[-1]),a[-1])[len(a)>2],'wb') as f:
                for c in r.iter_content(chunk_size=512):
                    n+=1
                    f.write(c)
                    prog(n/float(i))
        except KeyboardInterrupt:
            print '\n!> download canceled.'
            remove(u)
            exit()

def runq(q):
    for i in q:
        worker=Thread(target=download,args=i)
        worker.start()
        worker.join()
        q.remove(i)