Merge pull request #391 from akuker/rdmark_wget_downloads

Use requests to download files instead of urllib

src/web/file_cmds.py

@@ -188,29 +188,23 @@ def download_file_to_iso(scsi_id, url):
     Takes int scsi_id and str url
     Returns dict with boolean status and str msg
     """
-    import urllib.request
+    from time import time
-    import urllib.error as error
-    import time
     from subprocess import run
 
     server_info = get_server_info()
 
     file_name = url.split("/")[-1]
-    tmp_ts = int(time.time())
+    tmp_ts = int(time())
     tmp_dir = "/tmp/" + str(tmp_ts) + "/"
     os.mkdir(tmp_dir)
     tmp_full_path = tmp_dir + file_name
     iso_filename = f"{server_info['image_dir']}/{file_name}.iso"
 
-    try:
+    req_proc = download_to_dir(url, tmp_dir)
-        urllib.request.urlretrieve(url, tmp_full_path)
+
-    except (error.URLError, error.HTTPError, error.ContentTooShortError, FileNotFoundError) as e:
+    if req_proc["status"] == False:
-        logging.error(str(e))
+        return {"status": False, "msg": req_proc["msg"]}
-        return {"status": False, "msg": str(e)}
-    except:
-        return {"status": False, "msg": "Unknown error occurred."}
 
-    # iso_filename = make_cd(tmp_full_path, None, None) # not working yet
     iso_proc = run(
         ["genisoimage", "-hfs", "-o", iso_filename, tmp_full_path], capture_output=True
     )
@@ -225,20 +219,25 @@ def download_to_dir(url, save_dir):
     Takes str url, str save_dir
     Returns dict with boolean status and str msg
     """
-    import urllib.request
+    import requests
-    import urllib.error as error
+    from pathlib import PurePath
-
+    file_name = PurePath(url).name
-    file_name = url.split("/")[-1]
+    logging.info(f"Making a request to download {url}")
-    full_path = f"{save_dir}/{file_name}"
 
     try:
-        urllib.request.urlretrieve(url, full_path)
+        req = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
-        return {"status": True, "msg": "Downloaded the URL"}
+    except requests.exceptions.RequestException as e:
-    except (error.URLError, error.HTTPError, error.ContentTooShortError, FileNotFoundError) as e:
+        logging.warning(f"Request failed: {str(e)}")
-        logging.error(str(e))
         return {"status": False, "msg": str(e)}
-    except:
+
-        return {"status": False, "msg": "Unknown error occurred."}
+    with open(f"{save_dir}/{file_name}", "wb") as download:
+        download.write(req.content)
+
+    logging.info(f"Response encoding: {req.encoding}")
+    logging.info(f"Response content-type: {req.headers['content-type']}")
+    logging.info(f"Response status code: {req.status_code}")
+
+    return {"status": True, "msg": f"{url} downloaded to {save_dir}"}
 
 
 def write_config(file_name):

src/web/web.py

@@ -502,7 +502,7 @@ def download_to_iso():
         flash(process["msg"], "error")
         return redirect(url_for("index"))
 
-    process_attach = attach_image(scsi_id, type="SCCD", image=process["file_name"])
+    process_attach = attach_image(scsi_id, device_type="SCCD", image=process["file_name"])
     if process_attach["status"] == True:
         flash(f"Attached to SCSI ID {scsi_id}")
         return redirect(url_for("index"))