Merge pull request #819 from nucleogenic/docker-dev-environment

Docker environment for development and testing

.dockerignore

@@ -0,0 +1,32 @@
+# Exclude all by default
+/*
+
+# Paths to include
+!/docker/rascsi/rascsi_wrapper.sh
+!/docker/rascsi/cfilesystem.patch
+!/docker/rascsi-web/start.sh
+!/doc
+!/python
+!/src
+!/test
+!/easyinstall.sh
+!/LICENCE
+!/lido-driver.img
+!/README.md
+
+# From .gitignore
+venv
+*.pyc
+core
+**/.idea
+.DS_Store
+*.swp
+__pycache__
+current
+rascsi_interface_pb2.py
+src/raspberrypi/hfdisk/
+*~
+messages.pot
+messages.mo
+s.sh
+*-backups

.gitignore

@@ -12,9 +12,14 @@ src/raspberrypi/hfdisk/
 messages.pot
 messages.mo
 
+docker/docker-compose.override.yml
+/docker/volumes/images/*
+!/docker/volumes/images/.gitkeep
+/docker/volumes/config/*
+!/docker/volumes/config/.gitkeep
+
 # temporary user files
 s.sh
 
 # temporary kicad files
 *-backups
-

docker/README.md

@@ -0,0 +1,115 @@
+# Docker Environment for Development and Testing
+
+⚠️ **Important:** The Docker environment is unable to connect to the RaSCSI board and is
+intended for development and testing purposes only. To setup RaSCSI on a Raspberry Pi
+refer to the [setup instructions](https://github.com/akuker/RASCSI/wiki/Setup-Instructions)
+on the wiki instead.
+
+## Introduction
+
+This documentation currently focuses on using Docker for developing and testing the web UI.
+
+Additions, amendments and contributions for additional workflows are most welcome.
+
+## Getting Started
+
+The easiest way to launch a new environment is to use Docker Compose.
+
+```
+cd docker
+docker compose up
+```
+
+Containers will be built and started for the RaSCSI server and the web UI.
+
+The web UI can be accessed at:
+
+* http://localhost:8080
+* https://localhost:8443
+
+To stop the containers, press *Ctrl + C*, or run `docker compose stop` 
+from another terminal.
+
+## Environment Variables
+
+The following environment variables are available when using Docker Compose:
+
+| Environment Variable | Default  |
+| -------------------- | -------- |
+| `OS_DISTRO`          | debian   |
+| `OS_VERSION`         | buster   |
+| `OS_ARCH`            | amd64    |
+| `WEB_HTTP_PORT`      | 8080     |
+| `WEB_HTTPS_PORT`     | 8443     |
+| `WEB_LOG_LEVEL`      | info     |
+| `RASCSI_HOST`        | rascsi   |
+| `RASCSI_PORT`        | 6868     |
+| `RASCSI_PASSWORD`    | *[None]* |
+| `RASCSI_LOG_LEVEL`   | debug    |
+
+**Examples:**
+
+Run Debian "bullseye":
+```
+OS_VERSION=bullseye docker compose up
+```
+
+Start the web UI with the log level set to debug:
+```
+WEB_LOG_LEVEL=debug docker compose up
+```
+
+## Volumes
+
+When using Docker Compose the following volumes will be mounted automatically:
+
+| Local Path              | Container Path           |
+| ----------------------- | ------------------------ |
+| docker/volumes/images/  | /home/pi/images/         |
+| docker/volumes/config/  | /home/pi/.config/rascsi/ |
+
+
+## How To
+
+### Rebuild Containers
+
+You should rebuild the container images after checking out a different version of
+RaSCSI or making changes which affect the environment at build time, e.g. 
+`easyinstall.sh`.
+
+```
+docker compose up --build
+```
+
+### Open a Shell on a Running Container
+
+Run the following command, replacing `[CONTAINER]` with `rascsi` or `rascsi_web`.
+
+```
+docker compose exec [CONTAINER] bash
+```
+
+### Setup Live Editing for the Web UI
+
+Use a `docker-compose.override.yml` to mount the local `python` directory to
+`/home/pi/RASCSI/python/` in the `rascsi_web` container.
+
+Any changes to *.py files on the host computer (i.e. in your IDE) will trigger
+the web UI process to be restarted in the container.
+
+**Example:**
+```
+services:
+  rascsi_web:
+    volumes:
+      - ../python:/home/pi/RASCSI/python:delegated
+```
+
+### Connect the Web UI to a Real RaSCSI
+
+This can be useful for testing, but there are some caveats, e.g. the RaSCSI and the
+web UI will be accessing separate `images` directories.
+
+```
+RASCSI_HOST=foo RASCSI_PASSWORD=bar docker compose up
+```

docker/docker-compose.override.yml.example

@@ -0,0 +1,4 @@
+services:
+  rascsi_web:
+    volumes:
+      - ../python:/home/pi/RASCSI/python:delegated

docker/docker-compose.yml

@@ -0,0 +1,57 @@
+services:
+  rascsi:
+    container_name: rascsi
+    image: rascsi:develop-${OS_DISTRO:-debian}-${OS_VERSION:-buster}-${OS_ARCH:-amd64}-standalone
+    pull_policy: never
+    build:
+      context: ..
+      dockerfile: docker/rascsi/Dockerfile
+      args:
+        - OS_DISTRO=${OS_DISTRO:-debian}
+        - OS_VERSION=${OS_VERSION:-buster}
+        - OS_ARCH=${OS_ARCH:-amd64}
+    volumes:
+      - ./volumes/images:/home/pi/images:delegated
+      - ./volumes/config:/home/pi/.config/rascsi:delegated
+    ports:
+      - "127.0.0.1:${RASCSI_PORT:-6868}:6868"
+    environment:
+     - RASCSI_PASSWORD=${RASCSI_PASSWORD:-}
+    init: true
+    command: [
+      "/usr/local/bin/rascsi_wrapper.sh",
+      "-L",
+      "${RASCSI_LOG_LEVEL:-trace}",
+      "-r",
+      "7",
+      "-F",
+      "/home/pi/images"
+    ]
+
+  rascsi_web:
+    container_name: rascsi_web
+    image: rascsi:develop-${OS_DISTRO:-debian}-${OS_VERSION:-buster}-${OS_ARCH:-amd64}-web
+    pull_policy: never
+    build:
+      context: ..
+      dockerfile: docker/rascsi-web/Dockerfile
+      args:
+        - OS_DISTRO=${OS_DISTRO:-debian}
+        - OS_VERSION=${OS_VERSION:-buster}
+        - OS_ARCH=${OS_ARCH:-amd64}
+    volumes:
+      - ./volumes/images:/home/pi/images:delegated
+      - ./volumes/config:/home/pi/.config/rascsi:delegated
+    ports:
+      - "127.0.0.1:${WEB_HTTP_PORT:-8080}:80"
+      - "127.0.0.1:${WEB_HTTPS_PORT:-8443}:443"
+    environment:
+     - RASCSI_PASSWORD=${RASCSI_PASSWORD:-}
+    init: true
+    command: [
+      "start.sh",
+      "--rascsi-host=${RASCSI_HOST:-rascsi}",
+      "--rascsi-port=${RASCSI_PORT:-6868}",
+      "--log-level=${WEB_LOG_LEVEL:-info}",
+      "--dev-mode"
+    ]

docker/rascsi-web/Dockerfile

@@ -0,0 +1,24 @@
+ARG OS_DISTRO=debian
+ARG OS_VERSION=buster
+ARG OS_ARCH=amd64
+FROM "${OS_ARCH}/${OS_DISTRO}:${OS_VERSION}"
+
+EXPOSE 80 443
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends sudo rsyslog procps
+
+RUN groupadd pi
+RUN useradd --create-home --shell /bin/bash -g pi pi
+RUN echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+WORKDIR /home/pi
+USER pi
+COPY --chown=pi:pi . RASCSI
+RUN cd RASCSI && ./easyinstall.sh --run_choice=11 --skip-token
+
+USER root
+RUN pip3 install watchdog
+COPY docker/rascsi-web/start.sh /usr/local/bin/start.sh
+RUN chmod +x /usr/local/bin/start.sh
+CMD ["/usr/local/bin/start.sh"]

docker/rascsi-web/start.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+if ! [[ -f "/home/pi/RASCSI/python/common/src/rascsi_interface_pb2.py" ]]; then
+    # Build rascsi_interface_pb2.py with the protobuf compiler
+    protoc \
+        -I=/home/pi/RASCSI/src/raspberrypi \
+        --python_out=/home/pi/RASCSI/python/common/src \
+        rascsi_interface.proto
+fi
+
+# Start Nginx service
+nginx
+
+# Pass args to web UI start script
+if [[ $RASCSI_PASSWORD ]]; then
+    /home/pi/RASCSI/python/web/start.sh "$@" --password=$RASCSI_PASSWORD
+else
+    /home/pi/RASCSI/python/web/start.sh "$@"
+fi

docker/rascsi/Dockerfile

@@ -0,0 +1,29 @@
+ARG OS_DISTRO=debian
+ARG OS_VERSION=buster
+ARG OS_ARCH=amd64
+FROM "${OS_ARCH}/${OS_DISTRO}:${OS_VERSION}"
+
+EXPOSE 6868
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends sudo rsyslog patch
+
+RUN groupadd pi
+RUN useradd --create-home --shell /bin/bash -g pi pi
+RUN echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+USER pi
+COPY --chown=pi:pi . /home/pi/RASCSI
+WORKDIR /home/pi/RASCSI
+
+# Workaround for Bullseye amd64 compilation error
+# https://github.com/akuker/RASCSI/issues/821
+RUN patch -p0 < docker/rascsi/cfilesystem.patch
+
+# Install RaSCSI standalone
+RUN ./easyinstall.sh --run_choice=10 --cores=`nproc` --skip-token
+
+USER root
+COPY docker/rascsi/rascsi_wrapper.sh /usr/local/bin/rascsi_wrapper.sh
+RUN chmod +x /usr/local/bin/rascsi_wrapper.sh
+CMD ["/usr/local/bin/rascsi_wrapper.sh", "-L", "trace", "-r", "7", "-F", "/home/pi/images"]

docker/rascsi/cfilesystem.patch

@@ -0,0 +1,24 @@
+--- src/raspberrypi/devices/cfilesystem.cpp	2022-09-08 12:07:14.000000000 +0100
++++ src/raspberrypi/devices/cfilesystem.cpp.patched	2022-09-08 12:12:55.000000000 +0100
+@@ -1075,12 +1075,15 @@
+ 			m_dirHuman.name[i] = ' ';
+ 	}
+ 
+-	for (i = 0; i < 10; i++) {
+-		if (p < m_pszHumanExt)
+-			m_dirHuman.add[i] = *p++;
+-		else
+-			m_dirHuman.add[i] = '\0';
+-	}
++	// This code causes a compilation error on Debian "bullseye" (amd64)
++	// https://github.com/akuker/RASCSI/issues/821
++	// 
++	// for (i = 0; i < 10; i++) {
++	// 	if (p < m_pszHumanExt)
++	// 		m_dirHuman.add[i] = *p++;
++	// 	else
++	// 		m_dirHuman.add[i] = '\0';
++	// }
+ 
+ 	if (*p == '.')
+ 		p++;

docker/rascsi/rascsi_wrapper.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+if [[ $RASCSI_PASSWORD ]]; then
+    TOKEN_FILE="/home/pi/.config/rascsi/rascsi_secret"
+    mkdir -p /home/pi/.config/rascsi || true
+    echo $RASCSI_PASSWORD > $TOKEN_FILE
+    chmod 700 $TOKEN_FILE
+    /usr/local/bin/rascsi "$@" -P $TOKEN_FILE
+else
+    /usr/local/bin/rascsi "$@"
+fi

easyinstall.sh

@@ -63,6 +63,7 @@ LIDO_DRIVER=$BASE/lido-driver.img
 GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 GIT_REMOTE=${GIT_REMOTE:-origin}
 TOKEN=""
+SECRET_FILE="$HOME/.config/rascsi/rascsi_secret"
 
 set -e
 
@@ -82,10 +83,40 @@ function sudoCheck() {
 
 # install all dependency packages for RaSCSI Service
 function installPackages() {
-    sudo apt-get update && sudo apt-get install git libspdlog-dev libpcap-dev \
+    sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y -qq \
-    genisoimage python3 python3-venv python3-dev python3-pip nginx \
+        build-essential \
-    libpcap-dev protobuf-compiler bridge-utils libev-dev libevdev2 unar \
+        git \
-    disktype libgmock-dev -y </dev/null
+        libspdlog-dev \
+        libpcap-dev \
+        libprotobuf-dev \
+        genisoimage \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-venv \
+        python3-setuptools \
+        python3-wheel \
+        nginx-light \
+        protobuf-compiler \
+        bridge-utils \
+        libev-dev \
+        libevdev2 \
+        unzip \
+        unar \
+        disktype \
+        libgmock-dev
+}
+
+# install Debian packges for RaSCSI standalone
+function installPackagesStandalone() {
+    sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y -qq \
+        build-essential \
+        libspdlog-dev \
+        libpcap-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        disktype \
+        libgmock-dev
 }
 
 # cache the pip packages
@@ -249,23 +280,34 @@ function backupRaScsiService() {
 
 # Offers the choice of enabling token-based authentication for RaSCSI
 function configureTokenAuth() {
+    if [[ -f "$HOME/.rascsi_secret" ]]; then
+        sudo rm "$HOME/.rascsi_secret"
+        echo "Removed (legacy) RaSCSI token file"
+    fi
+
+    if [[ -f $SECRET_FILE ]]; then
+        sudo rm "$SECRET_FILE"
+        echo "Removed RaSCSI token file"
+    fi
+
+    if [[ $SKIP_TOKEN ]]; then
+        echo "Skipping RaSCSI token setup"
+        return 0
+    fi
+
+    if [[ -z $TOKEN ]]; then
         echo ""
         echo "Do you want to protect your RaSCSI installation with a password? [y/N]"
         read REPLY
 
-    SECRET_FILE="$HOME/.config/rascsi/rascsi_secret"
+        if ! [[ $REPLY =~ ^[Yy]$ ]]; then
+            return 0
+        fi
 
-    if [ "$REPLY" == "y" ] || [ "$REPLY" == "Y" ]; then
         echo -n "Enter the password that you want to use: "
         read -r TOKEN
-        if [ -f "$HOME/.rascsi_secret" ]; then
-            sudo rm "$HOME/.rascsi_secret"
-            echo "Removed old RaSCSI token file"
-        fi
-        if [ -f "$SECRET_FILE" ]; then
-            sudo rm "$SECRET_FILE"
-            echo "Removed old RaSCSI token file"
     fi
+
     echo "$TOKEN" > "$SECRET_FILE"
 
 	# Make the secret file owned and only readable by root
@@ -274,7 +316,6 @@ function configureTokenAuth() {
     echo ""
     echo "Configured RaSCSI to use $SECRET_FILE for authentication. This file is readable by root only."
     echo "Make note of your password: you will need it to use rasctl and other RaSCSI clients."
-    fi
 }
 
 # Modifies and installs the rascsi service
@@ -1202,8 +1243,9 @@ function runChoice() {
               echo "- Install manpages to /usr/local/man"
               sudoCheck
               createImagesDir
+              configureTokenAuth
               updateRaScsiGit
-              installPackages
+              installPackagesStandalone
               stopRaScsi
               compileRaScsi
               installRaScsi
@@ -1221,6 +1263,7 @@ function runChoice() {
               echo "- Create a self-signed certificate in /etc/ssl"
               sudoCheck
               createCfgDir
+              configureTokenAuth
               updateRaScsiGit
               installPackages
               preparePythonCommon
@@ -1294,24 +1337,38 @@ while [ "$1" != "" ]; do
     VALUE=$(echo "$1" | awk -F= '{print $2}')
     case $PARAM in
         -c | --connect_type)
+            if ! [[ $VALUE =~ ^(FULLSPEC|STANDARD|AIBOM|GAMERNIUM)$ ]]; then
+                echo "ERROR: The connect type parameter must have a value of: FULLSPEC, STANDARD, AIBOM or GAMERNIUM"
+                exit 1
+            fi
             CONNECT_TYPE=$VALUE
             ;;
         -r | --run_choice)
+            if ! [[ $VALUE =~ ^[1-9][0-9]?$ && $VALUE -ge 1 && $VALUE -le 12 ]]; then
+                echo "ERROR: The run choice parameter must have a numeric value between 1 and 12"
+                exit 1
+            fi
             RUN_CHOICE=$VALUE
             ;;
         -j | --cores)
+            if ! [[ $VALUE =~ ^[1-9][0-9]?$ ]]; then
+                echo "ERROR: The cores parameter must have a numeric value of at least 1"
+                exit 1
+            fi
             CORES=$VALUE
             ;;
-        *)
+        -t | --token)
-            echo "ERROR: unknown parameter \"$PARAM\""
+            if [[ -z $VALUE ]]; then
+                echo "ERROR: The token parameter cannot be empty"
                 exit 1
+            fi
+            TOKEN=$VALUE
             ;;
-    esac
+        -s | --skip-token)
-    case $VALUE in
+            SKIP_TOKEN=1
-        FULLSPEC | STANDARD | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12)
             ;;
         *)
-            echo "ERROR: unknown option \"$VALUE\""
+            echo "ERROR: Unknown parameter \"$PARAM\""
             exit 1
             ;;
     esac

python/common/src/util/unarchiver.py

@@ -10,6 +10,7 @@ import pathlib
 from tempfile import TemporaryDirectory
 from re import escape, match
 from json import loads, JSONDecodeError
+from shutil import move
 from util.run import run
 
 FORK_OUTPUT_TYPE_VISIBLE = "visible"
@@ -140,7 +141,7 @@ def extract_archive(file_path, **kwargs):
                 # The parent dir may not be specified as a member, so ensure it exists
                 target_path.parent.mkdir(parents=True, exist_ok=True)
                 logging.debug("Moving temp file: %s -> %s", source_path, target_path)
-                source_path.rename(target_path)
+                move(source_path, target_path)
                 moved.append(member)
 
             return {

python/web/requirements.txt

@@ -4,7 +4,7 @@ Flask==2.0.1
 itsdangerous==2.0.1
 Jinja2==3.0.1
 MarkupSafe==2.0.1
-protobuf==3.17.3
+protobuf==3.20.1
 requests==2.26.0
 simplepam==0.1.5
 flask_babel==2.0.0

python/web/src/web.py

@@ -1048,6 +1048,11 @@ if __name__ == "__main__":
         help="Log level for Web UI. Default: warning",
         choices=["debug", "info", "warning", "error", "critical"],
         )
+    parser.add_argument(
+        "--dev-mode",
+        action="store_true",
+        help="Run in development mode"
+        )
 
     arguments = parser.parse_args()
     APP.config["TOKEN"] = arguments.password
@@ -1064,5 +1069,14 @@ if __name__ == "__main__":
                         format="%(asctime)s %(levelname)s %(filename)s:%(lineno)s %(message)s",
                         level=arguments.log_level.upper())
 
+    if arguments.dev_mode:
+        print("Running rascsi-web in development mode ...")
+        APP.debug = True
+        from werkzeug.debug import DebuggedApplication
+        try:
+            bjoern.run(DebuggedApplication(APP, evalex=False), "0.0.0.0", arguments.port)
+        except KeyboardInterrupt:
+            pass
+    else:
         print("Serving rascsi-web...")
         bjoern.run(APP, "0.0.0.0", arguments.port)

python/web/start.sh

@@ -110,6 +110,9 @@ while [ "$1" != "" ]; do
     -l | --log-level)
         ARG_LOG_LEVEL="--log-level $VALUE"
         ;;
+    -d | --dev-mode)
+        ARG_DEV_MODE="--dev-mode"
+        ;;
     *)
         echo "ERROR: unknown parameter \"$PARAM\""
         exit 1
@@ -122,4 +125,10 @@ PYTHON_COMMON_PATH=$(dirname $PWD)/common/src
 echo "Starting web server for RaSCSI Web Interface..."
 export PYTHONPATH=$PWD/src:${PYTHON_COMMON_PATH}
 cd src
-python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL}
+
+if [[ $ARG_DEV_MODE ]]; then
+    watchmedo auto-restart --directory=../../ --pattern=*.py --recursive -- \
+    python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL} ${ARG_DEV_MODE}
+else
+    python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL} ${ARG_DEV_MODE}
+fi