download zip file from url, python sample code
from bs4 import BeautifulSoup
import requests
import os
import sys
def downloadZip(url, prefix_url, outpath):
mbyte=1024*1024
html = requests.get(url).text
soup = BeautifulSoup(html, features='lxml')
for name in soup.findAll('a', href=True):
#find A tag
zipurl = name['href']
#find file extension
if( zipurl.endswith('.zip') ):
#make download path
outfname = outpath +'/'+ zipurl.split('/')[-1]
#make url
zipurl = prefix_url+zipurl #http://aaa.com/ + 'abc.zip'
print(zipurl)
r = requests.get(zipurl, stream=True)
if( r.status_code == requests.codes.ok ) :
fsize = int(r.headers['content-length'])
print('Downloading %s (%sMb)'%(outfname, fsize/mbyte))
with open(outfname, 'wb') as fd:
for chunk in r.iter_content(chunk_size=1024): # chuck size can be larger
if chunk: # ignore keep-alive requests
fd.write(chunk)
fd.close()
base_path = os.getcwd()
path_join = os.path.join(base_path, 'data_download_pdf')
sys.path.append(path_join)
# point to output directory
outpath = path_join
url = 'https://www.gsa.gov/real-estate/real-estate-services/leasing-policy-procedures/lease-documents/lease-documents-region-1-new-england/'
prefix_url = 'https://www.gsa.gov/cdnstatic'
downloadZip(url, prefix_url, outpath)
