Sinisterly
Download System using Requests - Printable Version

+- Sinisterly (https://sinister.ly)
+-- Forum: Coding (https://sinister.ly/Forum-Coding)
+--- Forum: Python (https://sinister.ly/Forum-Python)
+--- Thread: Download System using Requests (/Thread-Download-System-using-Requests)



Download System using Requests - Inori - 12-16-2015

Using the super popular python module requests, which makes http, POST, and RESTful requests and traversal humanly possible (screw you, urllib2), you can 'get' a raw file from the url source and download it chunk by chunk as you would with raw html to use requests normally.

This project is fairly bug-free, save for the download bar not completing on some operations, but other than that, it's fully functional and works pretty well.

(Full code on github)

Spoiler: source code
__main__.py (used to execute code from the parent directory):
Code:
from internals import *

cmds=['e','q','r','v']
q=[]

def main():
    a=raw_input('~> ').split()
    if a[0] in cmds:
        if a[0]=='e':
            exit()
        elif a[0]=='q':
            if len(a)<2:
                print '!> No file supplied'
            else:
                uri=a[1]
                x=requests.get(uri)
                name=(a[1],uri.split('/')[-1])[len(a)<2]
                i=0
                if x.status_code==200:
                    for c in x: i+=1
                    q.append((name,uri.split('/')[2],requests.get(uri,stream=True),i,a))
                else:
                    print'#> %s returned status code %s'%(uri.split('/')[2],r.status_code)
        elif a[0]=='r':
            while len(q)>0:
                runq(q)
            print '\n#> download(s) complete.'
        elif a[0]=='v':
            for i in range(len(q)):
                print '#%s: %s,%s'%(i+1,q[i][-1][1],q[i][-1][2])

    else:
        print '!> %s is not a valid command'%a[0]
    main()

if tests():
    main()

internals.py:
Code:
import requests
from sys import stdout
from os import remove,makedirs,path
from threading import Thread

def tests():
    if not path.exists('./PyDownload/Downloads'):
        makedirs('./PyDownload/Downloads')
    try:
        requests.get('https://google.com').status_code
        return True
    except requests.exceptions.ConnectionError:
        print '#> Currently disconnected, try again later.'
        return False

def prog(p):
    block=int(round(30*p))
    text="\r#> [{0}] {1}% ".format("*"*block+" "*(30-block),str(p*100)[:5])
    stdout.write(text)
    stdout.flush()

def download(u,s,r,i,a,n=0):
    try:
        open(u,'r').close()
        print '!> file %s already exists'%u
    except IOError:
        try:
            print '#> downloading file %s from %s'%(u.split('/')[-1],s)
            with open('./PyDownload/Downloads/%s'%((u.split('/')[-1])+'.html'*('.' not in u.split('/')[-1]),a[-1])[len(a)>2],'wb') as f:
                for c in r.iter_content(chunk_size=512):
                    n+=1
                    f.write(c)
                    prog(n/float(i))
        except KeyboardInterrupt:
            print '\n!> download canceled.'
            remove(u)
            exit()

def runq(q):
    for i in q:
        worker=Thread(target=download,args=i)
        worker.start()
        worker.join()
        q.remove(i)