Macintosh-HW/RASCSI
1
0
Fork 0
mirror of https://github.com/akuker/RASCSI.git synced 2024-06-14 14:29:33 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #398 from akuker/chunk_download

Browse Source 
Do chunked downloads to avoid running out of memory
This commit is contained in:
Eric Helgeson 2021-10-28 08:56:07 -05:00 committed by GitHub
commit 670ff36bd2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/web/file_cmds.py
View File

@ -224,13 +224,15 @@ def download_to_dir(url, save_dir):
logging.info(f"Making a request to download {url}")
try:
req = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
with requests.get(url, stream=True, headers={"User-Agent": "Mozilla/5.0"}) as req:
req.raise_for_status()
with open(f"{save_dir}/{file_name}", "wb") as download:
for chunk in req.iter_content(chunk_size=8192):
download.write(chunk)
except requests.exceptions.RequestException as e:
logging.warning(f"Request failed: {str(e)}")
return {"status": False, "msg": str(e)}
with open(f"{save_dir}/{file_name}", "wb") as download:
download.write(req.content)
logging.info(f"Response encoding: {req.encoding}")
logging.info(f"Response content-type: {req.headers['content-type']}")