Speed up web uploads and various fixes (#158)

* Move to bjoern

* Stream upload

* wip

* Streams

* check for filename

* Allow easyinstall to be scripted

* Tell user we're done

* show help

* ignore hfdisk

* fix var

* Allow override of remote by setting GIT_REMOTE, default to origin

* Better handling of local changes

* Give user more info

* Fix #156 - show all supported types in web interface

* *.mos does not seem to be valid - ractl throws error

* Try some minor optomizations
Error handling

.gitignore

@@ -7,3 +7,4 @@ core
 __pycache__
 src/web/current
 src/oled_monitor/current
+src/raspberrypi/hfdisk/

easyinstall.sh

@@ -25,6 +25,7 @@ HFS_FORMAT=/usr/bin/hformat
 HFDISK_BIN=/usr/bin/hfdisk
 LIDO_DRIVER=~/RASCSI/lido-driver.img
 GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+GIT_REMOTE=${GIT_REMOTE:-origin}
 
 function initialChecks() {
     currentUser=$(whoami)
@@ -41,7 +42,7 @@ function initialChecks() {
 }
 
 function installPackages() {
-    sudo apt-get update && sudo apt install git libspdlog-dev libpcap-dev genisoimage python3 python3-venv nginx libpcap-dev protobuf-compiler bridge-utils -y
+    sudo apt-get update && sudo apt install git libspdlog-dev libpcap-dev genisoimage python3 python3-venv nginx libpcap-dev protobuf-compiler bridge-utils python3-dev libev-dev libevdev2 -y
 }
 
 # install all dependency packages for RaSCSI Service
@@ -102,12 +103,22 @@ function installRaScsiWebInterface() {
 }
 
 function updateRaScsiGit() {
-    echo "Updating checked out branch $GIT_BRANCH"
+    echo "Updating checked out branch $GIT_REMOTE/$GIT_BRANCH"
     cd ~/RASCSI
-    git fetch origin
-    git stash
-    git rebase origin/$GIT_BRANCH
-    git stash apply
+    stashed=0
+    if [[ $(git diff --stat) != '' ]]; then
+        echo 'There are local changes, we will stash and reapply them.'
+        git stash
+        stashed=1
+    fi
+
+    git fetch $GIT_REMOTE
+    git rebase $GIT_REMOTE/$GIT_BRANCH
+
+    if [ $stashed -eq 1 ]; then
+        echo "Reapplying local changes..."
+        git stash apply
+    fi
 }
 
 function updateRaScsi() {
@@ -128,6 +139,7 @@ function updateRaScsiWebInterface() {
     updateRaScsiGit
     sudo cp -f ~/RASCSI/src/web/service-infra/nginx-default.conf /etc/nginx/sites-available/default
     sudo cp -f ~/RASCSI/src/web/service-infra/502.html /var/www/html/502.html
+    echo "Restarting rascsi-web services..."
     sudo systemctl restart rascsi-web
     sudo systemctl restart nginx
 }
@@ -150,7 +162,7 @@ function createDriveCustom() {
         read driveName
     done
 
-    createDrive $driveSize "$driveName"
+    createDrive "$driveSize" "$driveName"
 }
 
 function formatDrive() {
@@ -177,7 +189,7 @@ function formatDrive() {
 
     if [ $partitionOk -eq 0 ]; then
         if [ ! -f $LIDO_DRIVER ];then
-            echo "Lido driver couldn't be found. Make sure RASCSI is up-to-date with git pull"
+            echo "Lido driver couldn't be found. Make sure RaSCSI is up-to-date with git pull"
             return 1
         fi
 
@@ -230,6 +242,75 @@ function createDrive() {
     fi
 }
 
+function runChoice() {
+  case $1 in
+          0)
+              echo "Installing RaSCSI Service + Web interface"
+              installRaScsi
+              installRaScsiWebInterface
+              createDrive600MB
+              showRaScsiStatus
+              echo "Installing RaSCSI Service + Web interface - Complete!"
+          ;;
+          1)
+              echo "Installing RaSCSI Service"
+              installRaScsi
+              showRaScsiStatus
+              echo "Installing RaSCSI Service - Complete!"
+          ;;
+          2)
+              echo "Installing RaSCSI Web interface"
+              installRaScsiWebInterface
+              echo "Installing RaSCSI Web interface - Complete!"
+          ;;
+          3)
+              echo "Updating RaSCSI Service + Web interface"
+              updateRaScsi
+              updateRaScsiWebInterface
+              showRaScsiStatus
+              echo "Updating RaSCSI Service + Web interface - Complete!"
+          ;;
+          4)
+              echo "Updating RaSCSI Service"
+              updateRaScsi
+              showRaScsiStatus
+              echo "Updating RaSCSI Service - Complete!"
+          ;;
+          5)
+              echo "Updating RaSCSI Web interface"
+              updateRaScsiWebInterface
+              echo "Updating RaSCSI Web interface - Complete!"
+          ;;
+          6)
+              echo "Creating a 600MB drive"
+              createDrive600MB
+              echo "Creating a 600MB drive - Complete!"
+          ;;
+          7)
+              echo "Creating a custom drive"
+              createDriveCustom
+              echo "Creating a custom drive - Complete!"
+          ;;
+          -h|--help|h|help)
+              showMenu
+          ;;
+          *)
+              echo "${1} is not a valid option, exiting..."
+              exit 1
+      esac
+}
+
+function readChoice() {
+   choice=-1
+
+   until [ $choice -ge "0" ] && [ $choice -le "7" ]; do
+       echo -n "Enter your choice (0-7) or CTRL-C to exit: "
+       read -r choice
+   done
+
+   runChoice "$choice"
+}
+
 function showMenu() {
     echo ""
     echo "Choose among the following options:"
@@ -244,60 +325,14 @@ function showMenu() {
     echo "CREATE EMPTY DRIVE"
     echo "  6) 600MB drive (recommended)"
     echo "  7) custom drive size (up to 4000MB)"
-
-
-    choice=-1
-
-    until [ $choice -ge "0" ] && [ $choice -le "7" ]; do
-        echo -n "Enter your choice (0-7) or CTRL-C to exit: "
-        read -r choice
-    done
-
-
-    case $choice in
-        0)
-            echo "Installing RaSCSI Service + Web interface"
-            installRaScsi
-            installRaScsiWebInterface
-            createDrive600MB
-            showRaScsiStatus
-        ;;
-        1)
-            echo "Installing RaSCSI Service"
-            installRaScsi
-            showRaScsiStatus
-        ;;
-        2)
-            echo "Installing RaSCSI Web interface"
-            installRaScsiWebInterface
-        ;;
-        3)
-            echo "Updating RaSCSI Service + Web interface"
-            updateRaScsi
-            updateRaScsiWebInterface
-            showRaScsiStatus
-        ;;
-        4)
-            echo "Updating RaSCSI Service"
-            updateRaScsi
-            showRaScsiStatus
-        ;;
-        5)
-            echo "Updating RaSCSI Web interface"
-            updateRaScsiWebInterface
-        ;;
-        6)
-            echo "Creating a 600MB drive"
-            createDrive600MB
-        ;;
-        7)
-            echo "Creating a custom drive"
-            createDriveCustom
-        ;;
-    esac
 }
 
 
 showRaSCSILogo
 initialChecks
-showMenu
+if [ -z "${1}" ]; then # $1 is unset, show menu
+    showMenu
+    readChoice
+else
+    runChoice "$1"
+fi

src/web/file_cmds.py

@@ -6,8 +6,8 @@ import time
 from ractl_cmds import attach_image
 from settings import *
 
-valid_file_types = ["*.hda", "*.iso", "*.cdr"]
-valid_file_types = r"|".join([fnmatch.translate(x) for x in valid_file_types])
+valid_file_suffix = ["*.hda", "*.hdn", "*.hdi", "*.nhd", "*.hdf", "*.hds", "*.iso", "*.cdr", "*.zip"]
+valid_file_types = r"|".join([fnmatch.translate(x) for x in valid_file_suffix])
 
 
 def create_new_image(file_name, type, size):

src/web/ractl_cmds.py

@@ -1,12 +1,12 @@
 import fnmatch
-import os
 import subprocess
 import re
 
 from settings import *
 
-valid_file_types = ["*.hda", "*.iso", "*.cdr", "*.zip"]
-valid_file_types = r"|".join([fnmatch.translate(x) for x in valid_file_types])
+
+valid_file_suffix = ["*.hda", "*.hdn", "*.hdi", "*.nhd", "*.hdf", "*.hds", "*.iso", "*.cdr", "*.zip"]
+valid_file_types = r"|".join([fnmatch.translate(x) for x in valid_file_suffix])
 # List of SCSI ID's you'd like to exclude - eg if you are on a Mac, the System is usually 7
 EXCLUDE_SCSI_IDS = [7]
 

src/web/requirements.txt

@@ -1,5 +1,6 @@
+bjoern==3.1.0
 click==7.1.2
-Flask==2.0.0
+Flask==2.0.1
 itsdangerous==2.0.0
 Jinja2==3.0.0
 machfs==1.2.4
@@ -7,6 +8,5 @@ macresources==1.2
 MarkupSafe==2.0.0
 rsrcfork==1.8.0
 waitress==1.4.4
-Werkzeug==2.0.0
 zope.event==4.5.0
 zope.interface==5.1.2

src/web/start.sh

@@ -37,6 +37,7 @@ if ! test -e venv; then
   echo "Activating venv"
   source venv/bin/activate
   echo "Installing requirements.txt"
+  pip install wheel
   pip install -r requirements.txt
   git rev-parse HEAD > current
 fi

src/web/templates/index.html

@@ -153,7 +153,7 @@
     <table style="border: none">
             <tr style="border: none">
                 <td style="border: none; vertical-align:top;">
-                <form action="/files/upload" method="post" enctype="multipart/form-data">
+                <form id="uploadForm" action="/files/upload/" onchange="fileSelect(event)" method="post" enctype="multipart/form-data">
                     <label for="file">File:</label>
                     <input type="file" name="file"/>
                     <input type="submit" value="Upload" />
@@ -161,6 +161,12 @@
             </td>
         </tr>
     </table>
+<script>
+    function fileSelect(e) {
+        document.getElementById("uploadForm").setAttribute('action', "/files/upload/" + e.target.files[0].name)
+        console.log(e.target.files[0].name);
+    }
+</script>
 
     <hr/>
 
@@ -216,7 +222,6 @@
                         <option value="nhd">SCSI Hard Disk image (T98Next HD image)</option>
                         <option value="hdf">SASI Hard Disk image (XM6 SASI HD image - typically only used with X68000)</option>
                         <option value="hds">SCSI Hard Disk image (XM6 SCSI HD image - typically only  used with X68000)</option>
-                        <option value="mos">SCSI Magneto-optical image (XM6 SCSI MO image - typically only used with X68000)</option>
                     </select>
                     <label for="size">Size(MB):</label>
                     <input type="number" placeholder="Size(MB)" name="size"/>

src/web/web.py

@@ -1,5 +1,7 @@
+import io
+import re
+
 from flask import Flask, render_template, request, flash, url_for, redirect, send_file
-from werkzeug.utils import secure_filename
 
 from file_cmds import (
     create_new_image,
@@ -22,6 +24,8 @@ from ractl_cmds import (
     daynaport_setup_bridge,
     list_config_files,
     detach_all,
+    valid_file_suffix,
+    valid_file_types
 )
 from settings import *
 
@@ -129,12 +133,15 @@ def attach():
     scsi_id = request.form.get("scsi_id")
 
     # Validate image type by suffix
-    if file_name.lower().endswith(".iso") or file_name.lower().endswith("iso"):
-        image_type = "cd"
-    elif file_name.lower().endswith(".hda"):
-        image_type = "hd"
+    print("file_name", file_name)
+    print("valid_file_types: ", valid_file_types)
+    if re.match(valid_file_types, file_name):
+        if file_name.lower().endswith("iso"):
+            image_type = "cd"
+        else:
+            image_type = "hd"
     else:
-        flash("Unknown file type. Valid files are .iso, .hda, .cdr", "error")
+        flash(f"Unknown file type. Valid files are: {', '.join(valid_file_suffix)}", "error")
         return redirect(url_for("index"))
 
     process = attach_image(scsi_id, file_name, image_type)
@@ -228,25 +235,35 @@ def download_img():
     return redirect(url_for("index"))
 
 
-@app.route("/files/upload", methods=["POST"])
-def upload_file():
-    if "file" not in request.files:
-        flash("No file part", "error")
+@app.route("/files/upload/<filename>", methods=["POST"])
+def upload_file(filename):
+    if not filename:
+        flash("No file provided.", "error")
         return redirect(url_for("index"))
-    file = request.files["file"]
-    if file:
-        filename = secure_filename(file.filename)
-        file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename))
-        return redirect(url_for("index", filename=filename))
+
+    file_path = os.path.join(app.config["UPLOAD_FOLDER"], filename)
+    if os.path.isfile(file_path):
+        flash(f"{filename} already exists.", "error")
+        return redirect(url_for("index"))
+
+    binary_new_file = "bx"
+    with open(file_path, binary_new_file, buffering=io.DEFAULT_BUFFER_SIZE) as f:
+        chunk_size = io.DEFAULT_BUFFER_SIZE
+        while True:
+            chunk = request.stream.read(chunk_size)
+            if len(chunk) == 0:
+                break
+            f.write(chunk)
+    return redirect(url_for("index", filename=filename))
 
 
 @app.route("/files/create", methods=["POST"])
 def create_file():
     file_name = request.form.get("file_name")
     size = request.form.get("size")
-    type = request.form.get("type")
+    file_type = request.form.get("type")
 
-    process = create_new_image(file_name, type, size)
+    process = create_new_image(file_name, file_type, size)
     if process.returncode == 0:
         flash("Drive created")
         return redirect(url_for("index"))
@@ -293,6 +310,7 @@ if __name__ == "__main__":
     os.makedirs(app.config["UPLOAD_FOLDER"], exist_ok=True)
     app.config["MAX_CONTENT_LENGTH"] = MAX_FILE_SIZE
 
-    from waitress import serve
+    import bjoern
+    print("Serving rascsi-web...")
+    bjoern.run(app, "0.0.0.0", 8080)
 
-    serve(app, host="0.0.0.0", port=8080)