From 500048ee7044a02095011c725f81f7a7bc6aab37 Mon Sep 17 00:00:00 2001 From: Daniel Markstedt Date: Thu, 28 Oct 2021 06:19:58 -0700 Subject: [PATCH] Do chunked downloads to avoid running out of memory --- src/web/file_cmds.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/web/file_cmds.py b/src/web/file_cmds.py index 9878b82d..516c0064 100644 --- a/src/web/file_cmds.py +++ b/src/web/file_cmds.py @@ -224,13 +224,15 @@ def download_to_dir(url, save_dir): logging.info(f"Making a request to download {url}") try: - req = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}) + with requests.get(url, stream=True, headers={"User-Agent": "Mozilla/5.0"}) as req: + req.raise_for_status() + with open(f"{save_dir}/{file_name}", "wb") as download: + for chunk in req.iter_content(chunk_size=8192): + download.write(chunk) except requests.exceptions.RequestException as e: logging.warning(f"Request failed: {str(e)}") return {"status": False, "msg": str(e)} - with open(f"{save_dir}/{file_name}", "wb") as download: - download.write(req.content) logging.info(f"Response encoding: {req.encoding}") logging.info(f"Response content-type: {req.headers['content-type']}")