Use the requests library to fetch files

This commit is contained in:
Daniel Markstedt 2021-10-27 11:11:37 -07:00
parent 46698c7b05
commit 7ec7d45c48

View File

@ -219,15 +219,23 @@ def download_to_dir(url, save_dir):
Takes str url, str save_dir Takes str url, str save_dir
Returns dict with boolean status and str msg Returns dict with boolean status and str msg
""" """
from subprocess import run import requests
from pathlib import PurePath
file_name = PurePath(url).name
logging.info(f"Making a request to download {url}")
wget_proc = run( try:
["wget", url, "-P", save_dir], capture_output=True req = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
) except requests.exceptions.RequestException as e:
if wget_proc.returncode != 0: logging.warning(f"Request failed: {str(e)}")
stderr = wget_proc.stderr.decode("utf-8") return {"status": False, "msg": str(e)}
logging.warning(f"Downloading failed: {stderr}")
return {"status": False, "msg": stderr} with open(f"{save_dir}/{file_name}", "wb") as download:
download.write(req.content)
logging.info(f"Response encoding: {req.encoding}")
logging.info(f"Response content-type: {req.headers['content-type']}")
logging.info(f"Response status code: {req.status_code}")
return {"status": True, "msg": f"{url} downloaded to {save_dir}"} return {"status": True, "msg": f"{url} downloaded to {save_dir}"}