Macintosh-HW/RASCSI
1
0
Fork 0
mirror of https://github.com/akuker/RASCSI.git synced 2024-12-21 08:29:59 +00:00
Code Issues Projects Releases Wiki Activity

Do chunked downloads to avoid running out of memory

Browse Source
This commit is contained in:
Daniel Markstedt 2021-10-28 06:19:58 -07:00
parent 3ff8f3ae61
commit 500048ee70
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/web/file_cmds.py
View File

@ -224,13 +224,15 @@ def download_to_dir(url, save_dir):
logging.info(f"Making a request to download {url}")
try:
req = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
with requests.get(url, stream=True, headers={"User-Agent": "Mozilla/5.0"}) as req:
req.raise_for_status()
with open(f"{save_dir}/{file_name}", "wb") as download:
for chunk in req.iter_content(chunk_size=8192):
download.write(chunk)
except requests.exceptions.RequestException as e:
logging.warning(f"Request failed: {str(e)}")
return {"status": False, "msg": str(e)}
with open(f"{save_dir}/{file_name}", "wb") as download:
download.write(req.content)
logging.info(f"Response encoding: {req.encoding}")
logging.info(f"Response content-type: {req.headers['content-type']}")