Formatted image file data structure that breaks down by subdir (#1102)

- New utility method for the web app, which sorts image files into dicts where the subdir is the key
- In the web ui, display each subdir in a table nested in a details tag.
- Allow for picking destination subdir when uploading files
- Style the expandable details blocks in the images table
- Add a check for ~ paths to the is_safe_path() utility method

python/web/src/static/themes/classic/style.css

@@ -104,6 +104,15 @@ ul.inline_list {
   list-style: none;
 }
 
+summary.dirname {
+  text-decoration: underline;
+  font-family: monospace;
+}
+
+summary.filename {
+  text-decoration: underline;
+}
+
 .dropzone, .dropzone * {
   box-sizing: border-box;
 }

python/web/src/static/themes/modern/style.css

@@ -53,7 +53,7 @@ input[type="radio"] {
   margin: 0 0.1rem 0 0.75rem;
 }
 
-div.noscriptmsg {
+div.notice {
   background: var(--danger);
   border-radius: var(--border-radius);
   padding: 0.5rem;
@@ -717,6 +717,16 @@ section#files p {
   margin-top: 1rem;
 }
 
+section#files details.subdir summary.dirname {
+  text-decoration: underline;
+  font-family: monospace;
+  margin: 0.5rem 0;
+}
+
+section#files details.contents summary.filename {
+  text-decoration: underline;
+}
+
 @media (max-width: 900px) {
   section#files table#images tr th:nth-child(2),
   section#files table#images tr td:nth-child(2) {

python/web/src/templates/index.html

@@ -209,6 +209,19 @@
     </ul>
 </details>
 
+{% if not files|length: %}
+<div class="notice">
+    {{ _("The images directory is currently empty.") }}
+</div>
+{% else %}
+
+<div>
+{% for subdir, group in formatted_image_files.items() %}
+
+<details class="subdir"{% if subdir == "images/" %} open{% endif %}>
+    <summary class="dirname">
+        {{ subdir }}
+    </summary>
 <table id="images" border="black" cellpadding="3" summary="List of files in the image directory">
     <tbody>
         <tr>
@@ -216,19 +229,12 @@
             <th scope="col">{{ _("Size") }}</th>
             <th scope="col">{{ _("Actions") }}</th>
         </tr>
-        {% if not files|length: %}
-            <tr class="directory-empty">
-                <td colspan="3">
-                    {{ _("The images directory is currently empty.") }}
-                </td>
-            </tr>
-        {% endif %}
-        {% for file in files|sort(attribute='name') %}
+        {% for file in group|sort(attribute='name') %}
         <tr>
             {% if file["prop"] %}
             <td>
-                <details>
-                    <summary>
+                <details class="contents">
+                    <summary class="filename">
                         {{ file["name"] }}
                     </summary>
                     <ul class="inline_list">
@@ -244,8 +250,8 @@
             </td>
             {% elif file["archive_contents"] %}
             <td>
-                <details>
-                    <summary>
+                <details class="contents">
+                    <summary class="filename">
                         {{ file["name"] }}
                     </summary>
                     <ul class="inline_list">
@@ -253,8 +259,8 @@
                             {% if not member["is_properties_file"] %}
                                 <li>
                                     {% if member["related_properties_file"] %}
-                                        <details>
-                                            <summary>
+                                        <details id="contents">
+                                            <summary class="filename">
                                                 <label>{{ member["path"] }}</label>
                                                 <form action="/files/extract_image" method="post" class="file-extract">
                                                     <input name="archive_file" type="hidden" value="{{ file['name'] }}">
@@ -361,6 +367,12 @@
         {% endfor %}
     </tbody>
 </table>
+{% if subdir != "/" %}
+</details>
+{% endif %}
+{% endfor %}
+</div>
+{% endif %}
 <p><small>{{ _("%(disk_space)s MiB disk space remaining on the system", disk_space=env["free_disk_space"]) }}</small></p>
 </section>
 
@@ -396,7 +408,7 @@
     <style type="text/css">
         section#upload { display: none; }
     </style>
-    <div class="noscriptmsg">
+    <div class="notice">
         {{ _("The file uploading functionality requires JavaScript.") }}
     </div>
 </noscript>

python/web/src/templates/upload.html

@@ -14,6 +14,14 @@
 <form name="dropper" action="/files/upload" method="post" class="dropzone dz-clickable" enctype="multipart/form-data" id="dropper">
     <input type="radio" name="destination" id="disk_images" value="disk_images" checked="checked">
     <label for="disk_images">{{ _("Disk Images") }}</label>
+    <select name="subdir" id="subdir">
+        <option value="">images/</option>
+    {% for subdir, group in formatted_image_files.items() %}
+    {% if subdir != "/" %}
+        <option value="{{subdir}}">images/{{subdir}}</option>
+    {% endif %}
+    {% endfor %}
+    </select>
     <input type="radio" name="destination" id="shared_files" value="shared_files">
     <label for="shared_files">{{ _("Shared Files") }}</label>
     <input type="radio" name="destination" id="piscsi_config" value="piscsi_config">

python/web/src/web.py

@@ -49,6 +49,7 @@ from web_utils import (
     map_device_types_and_names,
     get_device_name,
     map_image_file_descriptions,
+    format_image_list,
     format_drive_properties,
     get_properties_by_drive_name,
     auth_active,
@@ -223,12 +224,7 @@ def index():
     image_files = file_cmd.list_images()
     config_files = file_cmd.list_config_files()
     ip_addr, host = sys_cmd.get_ip_and_host()
-
-    extended_image_files = []
-    for image in image_files["files"]:
-        if image["detected_type"] != "UNDEFINED":
-            image["detected_type_name"] = device_types[image["detected_type"]]["name"]
-        extended_image_files.append(image)
+    formatted_image_files = format_image_list(image_files["files"], device_types)
 
     attached_images = []
     units = 0
@@ -266,7 +262,8 @@ def index():
         bridge_configured=sys_cmd.is_bridge_setup(),
         devices=formatted_devices,
         attached_images=attached_images,
-        files=extended_image_files,
+        formatted_image_files=formatted_image_files,
+        files=image_files["files"],
         config_files=config_files,
         device_types=device_types,
         scan_depth=server_info["scan_depth"],
@@ -317,10 +314,13 @@ def upload_page():
     """
     Sets up the data structures and kicks off the rendering of the file uploading page
     """
+    image_files = file_cmd.list_images()
+    formatted_image_files = format_image_list(image_files["files"])
 
     return response(
         template="upload.html",
         page_title=_("PiSCSI File Upload"),
+        formatted_image_files=formatted_image_files,
         max_file_size=int(int(MAX_FILE_SIZE) / 1024 / 1024),
         CFG_DIR=CFG_DIR,
         FILE_SERVER_DIR=FILE_SERVER_DIR,
@@ -990,15 +990,19 @@ def upload_file():
         return make_response(auth["msg"], 403)
 
     destination = request.form.get("destination")
+    subdir = request.form.get("subdir")
     if destination == "disk_images":
+        safe_path = is_safe_path(Path(subdir))
+        if not safe_path["status"]:
+            return make_response(safe_path["msg"], 403)
         server_info = piscsi_cmd.get_server_info()
-        destination_dir = server_info["image_dir"]
+        destination_dir = Path(server_info["image_dir"]) / subdir
     elif destination == "shared_files":
         destination_dir = FILE_SERVER_DIR
     elif destination == "piscsi_config":
         destination_dir = CFG_DIR
     else:
-        return make_response("Invalid destination", 403)
+        return make_response("Unknown destination", 403)
 
     return upload_with_dropzonejs(destination_dir)
 

python/web/src/web_utils.py

@@ -7,6 +7,7 @@ from grp import getgrall
 from os import path
 from pathlib import Path
 from ua_parser import user_agent_parser
+from re import findall
 
 from flask import request, make_response
 from flask_babel import _
@@ -146,6 +147,33 @@ def get_image_description(file_suffix):
     return file_suffix
 
 
+def format_image_list(image_files, device_types=None):
+    """
+    Takes a (list) of (dict) image_files and optional (list) device_types
+    Returns a formatted (dict) with groups of image_files per subdir key
+    """
+
+    root_image_files = []
+    subdir_image_files = {}
+    for image in image_files:
+        if (image["detected_type"] != "UNDEFINED") and device_types:
+            image["detected_type_name"] = device_types[image["detected_type"]]["name"]
+        subdir_path = findall("^.*/", image["name"])
+        if subdir_path:
+            subdir = subdir_path[0]
+            if subdir in subdir_image_files.keys():
+                subdir_image_files[f"images/{subdir}"].append(image)
+            else:
+                subdir_image_files[f"images/{subdir}"] = [image]
+        else:
+            root_image_files.append(image)
+
+    formatted_image_files = dict(sorted(subdir_image_files.items()))
+    if root_image_files:
+        formatted_image_files["images/"] = root_image_files
+    return formatted_image_files
+
+
 def format_drive_properties(drive_properties):
     """
     Takes a (dict) with structured drive properties data
@@ -256,10 +284,10 @@ def is_safe_path(file_name):
     Returns True if the path is safe
     Returns False if the path is either absolute, or tries to traverse the file system
     """
-    if file_name.is_absolute() or ".." in str(file_name):
+    if file_name.is_absolute() or ".." in str(file_name) or str(file_name)[0] == "~":
         return {
             "status": False,
-            "msg": _("%(file_name)s is not a valid path", file_name=file_name),
+            "msg": _("No permission to use path '%(file_name)s'", file_name=file_name),
         }
 
     return {"status": True, "msg": ""}

python/web/tests/api/test_files.py

@@ -254,6 +254,7 @@ def test_upload_file(http_client, delete_file):
 
             form_data = {
                 "destination": "disk_images",
+                "subdir": "",
                 "dzuuid": str(uuid.uuid4()),
                 "dzchunkindex": chunk_number,
                 "dzchunksize": chunk_size,