Run web API test suite in GitHub Actions (#1009)

- Fixed ignore patterns in .dockerignore
- Added healthchecks to backend and web containers
- Reduced Docker image sizes
- Removed RaSCSI references in various areas (e.g. rascsi -> backend)
- Added compilation-only step to easyinstall.sh
- Moved apt package lists to variables
- Revert to triggering GitHub Actions runs on push
- Updated web/frontend_checks workflow to run black and flake8 against all Python sources
- Capture log files from backend/web containers
- Fix None to float conversion bug when user agent is absent or unrecognised

.dockerignore

@@ -2,9 +2,8 @@
 /*
 
 # Paths to include
-!/docker/rascsi/rascsi_wrapper.sh
+!/docker/backend/rascsi_wrapper.sh
-!/docker/rascsi/cfilesystem.patch
+!/docker/web/start.sh
-!/docker/rascsi-web/start.sh
 !/doc
 !/python
 !/cpp
@@ -13,19 +12,27 @@
 !/LICENCE
 !/README.md
 
-# From .gitignore
+# Dev artifacts to exclude
-venv
+**/.git
-*.pyc
+
-core
+/cpp/bin
+/cpp/obj
+
+**/venv*
+**/*.pyc
+**/__pycache__
+**/.pytest_cache
+**/rascsi_interface_pb2.py
+**/report.xml
+
 **/.idea
-.DS_Store
+**/.vscode
-*.swp
+**/.DS_Store
-__pycache__
+
-current
+**/core
-rascsi_interface_pb2.py
+**/*.swp
-src/raspberrypi/hfdisk/
+**/current
-*~
+
-messages.pot
+**/node_modules
-messages.mo
+**/messages.pot
-s.sh
+**/messages.mo
-*-backups

.github/workflows/web.yml

@@ -2,22 +2,18 @@ name: Web Tests/Analysis
 
 on:
   workflow_dispatch:
-  pull_request:
+  push:
-    types: [opened, synchronize]
     paths:
       - 'python/web/**'
       - 'python/common/**'
       - '.github/workflows/web.yml'
-  push:
-    branches:
-      - develop
 
 jobs:
   backend_checks:
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: python/web
+        working-directory: python
     steps:
       - uses: actions/checkout@v3
 
@@ -26,14 +22,88 @@ jobs:
           python-version: 3.7.15
           cache: 'pip'
 
-      - run: pip install -r requirements-dev.txt
+      - run: pip install -r web/requirements-dev.txt
         id: pip
 
-      - run: black --check tests
+      - run: black --check .
 
-      - run: flake8 tests
+      - run: flake8 .
         if: success() || failure() && steps.pip.outcome == 'success'
 
+  backend_tests:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: docker
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Check DockerHub for existing backend image
+        run: |
+          export DOCKER_BACKEND_IMAGE="piscsi/backend-standalone:`git ls-files -s python .github/workflows/web.yml | git hash-object --stdin`"
+          echo "DOCKER_BACKEND_IMAGE=${DOCKER_BACKEND_IMAGE}" >> $GITHUB_ENV
+          docker pull --quiet ${DOCKER_BACKEND_IMAGE} || echo "DOCKER_BACKEND_NEEDS_PUSH=1" >> $GITHUB_ENV
+
+      - name: Build and launch containers
+        run: docker compose -f docker-compose.ci.yml up -d
+
+      - name: Run test suite
+        run: docker compose -f docker-compose.ci.yml run pytest -v
+
+      - name: Check if DockerHub secrets defined
+        run: if [[ $DOCKERHUB_USERNAME && $DOCKERHUB_TOKEN ]]; then echo "DOCKERHUB_SECRETS_DEFINED=1" >> $GITHUB_ENV; fi
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        if: env.DOCKERHUB_SECRETS_DEFINED && env.DOCKER_BACKEND_NEEDS_PUSH
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Push backend image to DockerHub
+        if: (success() || failure()) && env.DOCKERHUB_SECRETS_DEFINED && env.DOCKER_BACKEND_NEEDS_PUSH
+        run: docker compose -f docker-compose.ci.yml push backend
+
+      - name: Upload test artifacts
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: pytest-output.zip
+          path: |
+            docker/volumes/pytest/report.xml
+            docker/volumes/pytest/pytest.log
+
+      - name: Output container logs
+        if: success() || failure()
+        run: |
+          docker compose -f docker-compose.ci.yml logs backend > backend.log
+          docker compose -f docker-compose.ci.yml logs web > web.log
+          docker compose -f docker-compose.ci.yml logs -t | sort -u -k 3 > combined.log
+
+      - name: Upload backend log
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: backend.log
+          path: docker/backend.log
+
+      - name: Upload web log
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: web.log
+          path: docker/web.log
+
+      - name: Upload combined log
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: combined.log
+          path: docker/combined.log
+
   frontend_checks:
     runs-on: ubuntu-latest
     defaults:

.gitignore

@@ -1,13 +1,15 @@
 venv
 *.pyc
+*.swp
+*.log
+*~
 core
 .idea/
+.vscode
 .DS_Store
-*.swp
 __pycache__
 current
 rascsi_interface_pb2.py
-*~
 messages.pot
 messages.mo
 report.xml

docker/README.md

@@ -35,14 +35,12 @@ from another terminal.
 The following environment variables are available when using Docker Compose:
 
 | Environment Variable | Default  |
-| -------------------- | -------- |
+| -------------------- |----------|
-| `OS_DISTRO`          | debian   |
 | `OS_VERSION`         | buster   |
-| `OS_ARCH`            | amd64    |
 | `WEB_HTTP_PORT`      | 8080     |
 | `WEB_HTTPS_PORT`     | 8443     |
 | `WEB_LOG_LEVEL`      | info     |
-| `RASCSI_HOST`        | rascsi   |
+| `RASCSI_HOST`        | backend  |
 | `RASCSI_PORT`        | 6868     |
 | `RASCSI_PASSWORD`    | *[None]* |
 | `RASCSI_LOG_LEVEL`   | debug    |
@@ -83,7 +81,7 @@ docker compose up --build
 
 ### Open a Shell on a Running Container
 
-Run the following command, replacing `[CONTAINER]` with `rascsi` or `rascsi_web`.
+Run the following command, replacing `[CONTAINER]` with `backend` or `web`.
 
 ```
 docker compose exec [CONTAINER] bash
@@ -92,7 +90,7 @@ docker compose exec [CONTAINER] bash
 ### Setup Live Editing for the Web UI
 
 Use a `docker-compose.override.yml` to mount the local `python` directory to
-`/home/pi/RASCSI/python/` in the `rascsi_web` container.
+`/home/pi/RASCSI/python/` in the `web` container.
 
 Any changes to *.py files on the host computer (i.e. in your IDE) will trigger
 the web UI process to be restarted in the container.
@@ -100,7 +98,7 @@ the web UI process to be restarted in the container.
 **Example:**
 ```
 services:
-  rascsi_web:
+  web:
     volumes:
       - ../python:/home/pi/RASCSI/python:delegated
 ```

docker/backend/Dockerfile

@@ -0,0 +1,36 @@
+ARG DEBIAN_FRONTEND=noninteractive
+
+FROM debian:bullseye AS build
+RUN apt-get update && apt-get install --assume-yes --no-install-recommends sudo
+RUN groupadd pi \
+    && useradd --create-home --shell /bin/bash -g pi pi \
+    && echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+USER pi
+WORKDIR /home/pi/RASCSI
+
+COPY --chown=pi:pi easyinstall.sh .
+COPY --chown=pi:pi cpp cpp
+COPY --chown=pi:pi doc doc
+RUN ./easyinstall.sh --run_choice=15 --cores=`nproc`
+
+FROM debian:bullseye-slim AS runner
+USER root
+WORKDIR /home/pi
+
+COPY --from=build /home/pi/RASCSI/cpp/bin/fullspec/* /usr/local/bin/
+COPY docker/backend/rascsi_wrapper.sh /usr/local/bin/rascsi_wrapper.sh
+RUN chmod +x /usr/local/bin/*
+RUN mkdir -p /home/pi/images
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends --assume-yes libpcap-dev libprotobuf-dev \
+    && apt autoremove -y \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+EXPOSE 6868
+ENTRYPOINT ["/usr/local/bin/rascsi_wrapper.sh", "-r", "7", "-F", "/home/pi/images"]
+CMD ["-L", "trace"]
+
+HEALTHCHECK --interval=5m --timeout=1s CMD rasctl -v

docker/docker-compose.ci.yml

@@ -0,0 +1,42 @@
+services:
+  backend:
+    image: ${DOCKER_BACKEND_IMAGE}
+    build:
+      context: ..
+      dockerfile: docker/backend/Dockerfile
+    init: true
+    volumes:
+      - ./volumes/images:/home/pi/images:delegated
+    healthcheck:
+      interval: 5s
+      start_period: 5s
+
+  web:
+    build:
+      context: ..
+      dockerfile: docker/web/Dockerfile
+      args:
+        - OS_VERSION=buster
+    volumes:
+      - ./volumes/images:/home/pi/images:delegated
+    init: true
+    command: ["--rascsi-host=backend", "--log-level=debug"]
+    healthcheck:
+      interval: 5s
+      start_period: 5s
+
+  pytest:
+    depends_on:
+      web:
+        condition: service_healthy
+      backend:
+        condition: service_healthy
+    profiles:
+      - webui-tests
+    build:
+      context: ..
+      dockerfile: docker/pytest/Dockerfile
+    working_dir: /src
+    volumes:
+      - ./volumes/pytest:/src/tests/output:delegated
+    command: ["-vv"]

docker/docker-compose.override.yml.example

@@ -1,4 +1,7 @@
 services:
-  rascsi_web:
+  web:
     volumes:
       - ../python:/home/pi/RASCSI/python:delegated
+  pytest:
+    volumes:
+      - ../python/web:/src:delegated

docker/docker-compose.yml

@@ -1,15 +1,11 @@
 services:
-  rascsi:
+  backend:
-    container_name: rascsi
+    container_name: rascsi_backend
-    image: rascsi:develop-${OS_DISTRO:-debian}-${OS_VERSION:-buster}-${OS_ARCH:-amd64}-standalone
+    image: rascsi-backend
     pull_policy: never
     build:
       context: ..
-      dockerfile: docker/rascsi/Dockerfile
+      dockerfile: docker/backend/Dockerfile
-      args:
-        - OS_DISTRO=${OS_DISTRO:-debian}
-        - OS_VERSION=${OS_VERSION:-buster}
-        - OS_ARCH=${OS_ARCH:-amd64}
     volumes:
       - ./volumes/images:/home/pi/images:delegated
       - ./volumes/config:/home/pi/.config/rascsi:delegated
@@ -19,26 +15,19 @@ services:
      - RASCSI_PASSWORD=${RASCSI_PASSWORD:-}
     init: true
     command: [
-      "/usr/local/bin/rascsi_wrapper.sh",
       "-L",
       "${RASCSI_LOG_LEVEL:-trace}",
-      "-r",
-      "7",
-      "-F",
-      "/home/pi/images"
     ]
 
-  rascsi_web:
+  web:
     container_name: rascsi_web
-    image: rascsi:develop-${OS_DISTRO:-debian}-${OS_VERSION:-buster}-${OS_ARCH:-amd64}-web
+    image: rascsi-web:${OS_VERSION:-buster}
     pull_policy: never
     build:
       context: ..
-      dockerfile: docker/rascsi-web/Dockerfile
+      dockerfile: docker/web/Dockerfile
       args:
-        - OS_DISTRO=${OS_DISTRO:-debian}
         - OS_VERSION=${OS_VERSION:-buster}
-        - OS_ARCH=${OS_ARCH:-amd64}
     volumes:
       - ./volumes/images:/home/pi/images:delegated
       - ./volumes/config:/home/pi/.config/rascsi:delegated
@@ -49,23 +38,20 @@ services:
      - RASCSI_PASSWORD=${RASCSI_PASSWORD:-}
     init: true
     command: [
-      "start.sh",
+      "--rascsi-host=${RASCSI_HOST:-backend}",
-      "--rascsi-host=${RASCSI_HOST:-rascsi}",
       "--rascsi-port=${RASCSI_PORT:-6868}",
-      "--log-level=${WEB_LOG_LEVEL:-info}",
+      "--log-level=${WEB_LOG_LEVEL:-debug}",
       "--dev-mode"
     ]
 
   pytest:
-    container_name: pytest
+    container_name: rascsi_pytest
-    image: rascsi:pytest
+    image: rascsi-pytest
     pull_policy: never
     profiles:
       - webui-tests
     build:
       context: ..
       dockerfile: docker/pytest/Dockerfile
-    volumes:
-      - ../python/web:/src:delegated
     working_dir: /src
-    entrypoint: "pytest"
+    command: ["-vv"]

docker/pytest/Dockerfile

@@ -1,5 +1,12 @@
-FROM python:3.7-bullseye
+FROM python:3.7-slim
 ENV DOCKER=1
 
-COPY python/web/requirements-dev.txt /requirements-dev.txt
+WORKDIR /src
-RUN pip install -r /requirements-dev.txt
+
+COPY python/web/requirements-dev.txt /src/requirements-dev.txt
+COPY python/web/pyproject.toml /src/pyproject.toml
+COPY python/web/tests /src/tests
+
+RUN pip install --no-cache-dir -r /src/requirements-dev.txt
+
+ENTRYPOINT ["pytest"]

docker/rascsi-web/Dockerfile

@@ -1,49 +0,0 @@
-ARG OS_DISTRO=debian
-ARG OS_VERSION=buster
-ARG OS_ARCH=amd64
-FROM "${OS_ARCH}/${OS_DISTRO}:${OS_VERSION}"
-
-EXPOSE 80 443
-
-ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-    sudo \
-    systemd \
-    rsyslog \
-    procps \
-    man-db \
-    wget \
-    git
-
-RUN groupadd pi
-RUN useradd --create-home --shell /bin/bash -g pi pi
-RUN echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
-RUN echo "pi:rascsi" | chpasswd
-
-# Allows custom PATH for mock commands to work when executing with sudo
-RUN sed -i 's/^Defaults\tsecure_path/#Defaults\tsecure_path./' /etc/sudoers
-
-RUN mkdir /home/pi/shared_files
-RUN touch /etc/dhcpcd.conf
-RUN mkdir -p /etc/network/interfaces.d/
-
-USER pi
-WORKDIR /home/pi/RASCSI
-COPY --chown=pi:pi . .
-
-# Install standalone RaSCSI web UI
-RUN ./easyinstall.sh --run_choice=11
-
-# Enable web UI authentication
-RUN ./easyinstall.sh --run_choice=13
-
-# Setup wired network bridge
-RUN ./easyinstall.sh --run_choice=5 --headless
-
-USER root
-WORKDIR /home/pi
-RUN pip3 install watchdog
-COPY docker/rascsi-web/start.sh /usr/local/bin/start.sh
-RUN chmod +x /usr/local/bin/start.sh
-CMD ["/usr/local/bin/start.sh"]

docker/rascsi/Dockerfile

@@ -1,26 +0,0 @@
-ARG OS_DISTRO=debian
-ARG OS_VERSION=buster
-ARG OS_ARCH=amd64
-FROM "${OS_ARCH}/${OS_DISTRO}:${OS_VERSION}"
-
-EXPOSE 6868
-
-ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends sudo systemd rsyslog patch wget
-
-RUN groupadd pi
-RUN useradd --create-home --shell /bin/bash -g pi pi
-RUN echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
-
-USER pi
-WORKDIR /home/pi/RASCSI
-COPY --chown=pi:pi . .
-
-# Install RaSCSI standalone
-RUN ./easyinstall.sh --run_choice=10 --cores=`nproc`
-
-USER root
-WORKDIR /home/pi
-COPY docker/rascsi/rascsi_wrapper.sh /usr/local/bin/rascsi_wrapper.sh
-RUN chmod +x /usr/local/bin/rascsi_wrapper.sh
-CMD ["/usr/local/bin/rascsi_wrapper.sh", "-L", "trace", "-r", "7", "-F", "/home/pi/images"]

docker/web/Dockerfile

@@ -0,0 +1,57 @@
+ARG DEBIAN_FRONTEND=noninteractive
+ARG OS_VERSION=buster
+
+FROM "debian:${OS_VERSION}-slim"
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends sudo systemd rsyslog procps man-db wget git \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN groupadd pi \
+    && useradd --create-home --shell /bin/bash -g pi pi \
+    && echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \
+    && echo "pi:rascsi" | chpasswd
+
+# Allows custom PATH for mock commands to work when executing with sudo
+RUN sed -i 's/^Defaults\tsecure_path/#Defaults\tsecure_path./' /etc/sudoers
+
+RUN mkdir -p /home/pi/shared_files \
+    && mkdir /home/pi/images \
+    && mkdir -p /etc/network/interfaces.d \
+    && touch /etc/dhcpcd.conf
+
+USER pi
+WORKDIR /home/pi/RASCSI
+
+RUN mkdir /home/pi/RASCSI/{python,cpp}
+COPY --chown=pi:pi easyinstall.sh .
+COPY --chown=pi:pi cpp/os_integration cpp/os_integration
+COPY --chown=pi:pi cpp/rascsi_interface.proto cpp/rascsi_interface.proto
+COPY --chown=pi:pi python/web python/web
+COPY --chown=pi:pi python/common python/common
+
+# Install standalone RaSCSI web UI
+RUN ./easyinstall.sh --run_choice=11 \
+    && sudo apt-get remove build-essential --yes \
+    && sudo apt autoremove -y \
+    && sudo apt-get clean \
+    && sudo rm -rf /var/lib/apt/lists/*
+
+# Enable web UI authentication
+RUN ./easyinstall.sh --run_choice=13
+
+# Setup wired network bridge
+RUN ./easyinstall.sh --run_choice=5 --headless
+
+USER root
+WORKDIR /home/pi
+RUN pip3 install --no-cache-dir PyYAML watchdog
+COPY docker/web/start.sh /usr/local/bin/start.sh
+RUN chmod +x /usr/local/bin/start.sh
+
+EXPOSE 80 443
+ENTRYPOINT ["/usr/local/bin/start.sh"]
+
+HEALTHCHECK --interval=5m --timeout=3s \
+  CMD wget --quiet --server-response http://localhost/healthcheck 2>&1 | grep "200 OK"

docker/web/start.sh

@@ -3,7 +3,7 @@
 if ! [[ -f "/home/pi/RASCSI/python/common/src/rascsi_interface_pb2.py" ]]; then
     # Build rascsi_interface_pb2.py with the protobuf compiler
     protoc \
-        -I=/home/pi/RASCSI/src/raspberrypi \
+        -I=/home/pi/RASCSI/cpp \
         --python_out=/home/pi/RASCSI/python/common/src \
         rascsi_interface.proto
 fi

easyinstall.sh

@@ -69,6 +69,11 @@ SECRET_FILE="$HOME/.config/rascsi/rascsi_secret"
 FILE_SHARE_PATH="$HOME/shared_files"
 FILE_SHARE_NAME="Pi File Server"
 
+APT_PACKAGES_COMMON="build-essential git protobuf-compiler bridge-utils"
+APT_PACKAGES_BACKEND="libspdlog-dev libpcap-dev libprotobuf-dev protobuf-compiler libgmock-dev clang-11"
+APT_PACKAGES_PYTHON="python3 python3-dev python3-pip python3-venv python3-setuptools python3-wheel libev-dev libevdev2"
+APT_PACKAGES_WEB="nginx-light genisoimage man2html hfsutils dosfstools kpartx unzip unar disktype"
+
 set -e
 
 # checks to run before entering the script main menu
@@ -96,51 +101,36 @@ function installPackages() {
         return 0
     fi
     sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y -qq \
-        build-essential \
+        $APT_PACKAGES_COMMON \
-        git \
+        $APT_PACKAGES_BACKEND \
-        libspdlog-dev \
+        $APT_PACKAGES_PYTHON \
-        libpcap-dev \
+        $APT_PACKAGES_WEB
-        libprotobuf-dev \
-        genisoimage \
-        python3 \
-        python3-dev \
-        python3-pip \
-        python3-venv \
-        python3-setuptools \
-        python3-wheel \
-        nginx-light \
-        protobuf-compiler \
-        bridge-utils \
-        libev-dev \
-        libevdev2 \
-        unzip \
-        unar \
-        disktype \
-        libgmock-dev \
-        man2html \
-        hfsutils \
-        dosfstools \
-        kpartx \
-        clang-11
 }
 
-# install Debian packges for RaSCSI standalone
+# install Debian packages for RaSCSI standalone
 function installPackagesStandalone() {
     if [[ $SKIP_PACKAGES ]]; then
         echo "Skipping package installation"
         return 0
     fi
     sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y -qq \
-        build-essential \
+        $APT_PACKAGES_COMMON \
-        git \
+        $APT_PACKAGES_BACKEND
-        libspdlog-dev \
-        libpcap-dev \
-        libprotobuf-dev \
-        protobuf-compiler \
-        libgmock-dev \
-        clang-11
 }
 
+# install Debian packages for RaSCSI web UI standalone
+function installPackagesWeb() {
+    if [[ $SKIP_PACKAGES ]]; then
+        echo "Skipping package installation"
+        return 0
+    fi
+    sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y -qq \
+        $APT_PACKAGES_COMMON \
+        $APT_PACKAGES_PYTHON \
+        $APT_PACKAGES_WEB
+}
+
+
 # cache the pip packages
 function cachePipPackages(){
     pushd $WEB_INSTALL_PATH
@@ -1363,7 +1353,7 @@ function runChoice() {
               sudoCheck
               createCfgDir
               updateRaScsiGit
-              installPackages
+              installPackagesWeb
               installHfdisk
               fetchHardDiskDrivers
               preparePythonCommon
@@ -1394,6 +1384,10 @@ function runChoice() {
               shareImagesWithNetatalk
               echo "Configuring AppleShare File Server - Complete!"
           ;;
+          15)
+              installPackagesStandalone
+              compileRaScsi
+          ;;
           -h|--help|h|help)
               showMenu
           ;;
@@ -1439,6 +1433,7 @@ function showMenu() {
     echo " 13) Enable or disable RaSCSI Web Interface authentication"
     echo "EXPERIMENTAL FEATURES"
     echo " 14) Share the images dir over AppleShare (requires Netatalk)"
+    echo " 15) Compile RaSCSI binaries"
 }
 
 # parse arguments passed to the script
@@ -1454,7 +1449,7 @@ while [ "$1" != "" ]; do
             CONNECT_TYPE=$VALUE
             ;;
         -r | --run_choice)
-            if ! [[ $VALUE =~ ^[1-9][0-9]?$ && $VALUE -ge 1 && $VALUE -le 14 ]]; then
+            if ! [[ $VALUE =~ ^[1-9][0-9]?$ && $VALUE -ge 1 && $VALUE -le 15 ]]; then
                 echo "ERROR: The run choice parameter must have a numeric value between 1 and 14"
                 exit 1
             fi

python/common/src/rascsi/file_cmds.py

@@ -47,6 +47,13 @@ class FileCmds:
         self.token = token
         self.locale = locale
 
+    def send_pb_command(self, command):
+        if logging.getLogger().isEnabledFor(logging.DEBUG):
+            # TODO: Uncouple/move to common dependency
+            logging.debug(self.ractl.format_pb_command(command))
+
+        return self.sock_cmd.send_pb_command(command.SerializeToString())
+
     # noinspection PyMethodMayBeStatic
     # pylint: disable=no-self-use
     def list_files(self, file_types, dir_path):
@@ -89,7 +96,7 @@ class FileCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
 
@@ -168,7 +175,7 @@ class FileCmds:
         command.params["size"] = str(size)
         command.params["read_only"] = "false"
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -186,7 +193,7 @@ class FileCmds:
 
         command.params["file"] = file_name
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -205,7 +212,7 @@ class FileCmds:
         command.params["from"] = file_name
         command.params["to"] = new_file_name
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -224,7 +231,7 @@ class FileCmds:
         command.params["from"] = file_name
         command.params["to"] = new_file_name
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}

python/common/src/rascsi/ractl_cmds.py

@@ -5,6 +5,7 @@ Module for commands sent to the RaSCSI backend service.
 import rascsi_interface_pb2 as proto
 from rascsi.return_codes import ReturnCodes
 from rascsi.socket_cmds import SocketCmds
+import logging
 
 
 class RaCtlCmds:
@@ -17,6 +18,12 @@ class RaCtlCmds:
         self.token = token
         self.locale = locale
 
+    def send_pb_command(self, command):
+        if logging.getLogger().isEnabledFor(logging.DEBUG):
+            logging.debug(self.format_pb_command(command))
+
+        return self.sock_cmd.send_pb_command(command.SerializeToString())
+
     def get_server_info(self):
         """
         Sends a SERVER_INFO command to the server.
@@ -35,7 +42,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         version = (
@@ -93,7 +100,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         scsi_ids = []
@@ -114,7 +121,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         ifs = result.network_interfaces_info.name
@@ -133,7 +140,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         device_types = {}
@@ -199,7 +206,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.token
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         images_dir = result.image_files_info.default_image_folder
@@ -273,7 +280,7 @@ class RaCtlCmds:
 
         command.devices.append(devices)
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -295,7 +302,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -310,7 +317,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -332,7 +339,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -360,7 +367,7 @@ class RaCtlCmds:
                 device.unit = int(unit)
             command.devices.append(device)
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
 
@@ -430,7 +437,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -447,7 +454,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -466,7 +473,7 @@ class RaCtlCmds:
         command.params["token"] = self.token
         command.params["locale"] = self.locale
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
@@ -480,7 +487,37 @@ class RaCtlCmds:
         command = proto.PbCommand()
         command.operation = proto.PbOperation.CHECK_AUTHENTICATION
 
-        data = self.sock_cmd.send_pb_command(command.SerializeToString())
+        data = self.send_pb_command(command)
         result = proto.PbResult()
         result.ParseFromString(data)
         return {"status": result.status, "msg": result.msg}
+
+    def format_pb_command(self, command):
+        """
+        Formats the Protobuf command for output
+        """
+        message = f"Sending: {proto.PbOperation.Name(command.operation)}"
+
+        params = {
+            name: "***" if name == "token" else value
+            for (name, value) in sorted(command.params.items())
+        }
+        message += f", params: {params}"
+
+        for device in command.devices:
+            formatted_device = {
+                key: value
+                for (key, value) in {
+                    "id": device.id,
+                    "unit": device.unit,
+                    "type": proto.PbDeviceType.Name(device.type) if device.type else None,
+                    "params": device.params,
+                    "vendor": device.vendor,
+                    "product": device.product,
+                    "revision": device.revision,
+                }.items()
+                if key == "id" or value
+            }
+            message += f", device: {formatted_device}"
+
+        return message

python/web/mock/bin/git

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exit 0

python/web/pyproject.toml

@@ -1,4 +1,4 @@
 [tool.pytest.ini_options]
-addopts = "--junitxml=report.xml"
+addopts = "--junitxml=tests/output/report.xml --log-file=tests/output/pytest.log"
 log_cli = true
 log_cli_level = "warn"

python/web/src/templates/index.html

@@ -709,7 +709,11 @@
     <label for="locale">{{ _("Language:") }}</label>
     <select name="locale" id="locale">
     {% for locale in locales %}
-        <option value="{{ locale.language }}">
+        {% if locale.language == env['locale'] %}
+            <option value="{{ locale.language }}" selected="selected">
+        {% else %}
+            <option value="{{ locale.language }}">
+        {% endif %}
         {{ locale.language }} - {{ locale.display_name }}
         </option>
     {% endfor %}

python/web/src/web.py

@@ -2,8 +2,8 @@
 Module for the Flask app rendering and endpoints
 """
 
-import sys
 import logging
+import logging.config
 import argparse
 from pathlib import Path, PurePath
 from functools import wraps
@@ -168,16 +168,16 @@ def get_locale():
     """
     Uses the session language, or tries to detect based on accept-languages header
     """
-    try:
+    session_locale = session.get("language")
-        language = session["language"]
+    if session_locale:
-    except KeyError:
+        return session_locale
-        language = ""
+
-        logging.info("The default locale could not be detected. Falling back to English.")
+    client_locale = request.accept_languages.best_match(LANGUAGES)
-    if language:
+    if client_locale:
-        return language
+        return client_locale
-    # Hardcoded fallback to "en" when the user agent does not send an accept-language header
+
-    language = request.accept_languages.best_match(LANGUAGES) or "en"
+    logging.info("The default locale could not be detected. Falling back to English.")
-    return language
+    return "en"
 
 
 def get_supported_locales():
@@ -994,7 +994,7 @@ def create_file():
 
     message_postfix = ""
 
-    # Formatting and injecting driver, if one is choosen
+    # Formatting and injecting driver, if one is chosen
     if drive_format:
         volume_name = f"HD {size / 1024 / 1024:0.0f}M"
         known_formats = [
@@ -1285,6 +1285,11 @@ def change_theme():
     return response(message=_("Theme changed to '%(theme)s'.", theme=theme))
 
 
+@APP.route("/healthcheck", methods=["GET"])
+def healthcheck():
+    return "", 200
+
+
 @APP.before_first_request
 def detect_locale():
     """
@@ -1296,6 +1301,22 @@ def detect_locale():
     file_cmd.locale = session["language"]
 
 
+@APP.before_request
+def log_http_request():
+    if logging.getLogger().isEnabledFor(logging.DEBUG):
+        message = f"HTTP request: {request.method} {request.path}"
+
+        if request.method == "POST":
+            if request.path == "/login":
+                message += " (<hidden>)"
+            elif len(request.get_data()) > 100:
+                message += f" (payload: {request.get_data()[:100]} <truncated>)"
+            else:
+                message += f" (payload: {request.get_data()})"
+
+        logging.debug(message)
+
+
 if __name__ == "__main__":
     APP.secret_key = "rascsi_is_awesome_insecure_secret_key"
     APP.config["SESSION_TYPE"] = "filesystem"
@@ -1347,6 +1368,27 @@ if __name__ == "__main__":
     arguments = parser.parse_args()
     APP.config["RASCSI_TOKEN"] = arguments.password
 
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "formatters": {
+                "default": {
+                    "format": "[%(asctime)s] [%(levelname)s] %(filename)s:%(lineno)s %(message)s",
+                }
+            },
+            "handlers": {
+                "wsgi": {
+                    "class": "logging.StreamHandler",
+                    "formatter": "default",
+                }
+            },
+            "root": {
+                "level": arguments.log_level.upper(),
+                "handlers": ["wsgi"],
+            },
+        }
+    )
+
     sock_cmd = SocketCmdsFlask(host=arguments.rascsi_host, port=arguments.rascsi_port)
     ractl_cmd = RaCtlCmds(sock_cmd=sock_cmd, token=APP.config["RASCSI_TOKEN"])
     file_cmd = FileCmds(sock_cmd=sock_cmd, ractl=ractl_cmd, token=APP.config["RASCSI_TOKEN"])
@@ -1365,14 +1407,9 @@ if __name__ == "__main__":
         APP.config["RASCSI_DRIVE_PROPERTIES"] = []
         logging.warning("Could not read drive properties from %s", DRIVE_PROPERTIES_FILE)
 
-    logging.basicConfig(
+    logging.info("Starting WSGI server...")
-        stream=sys.stdout,
-        format="%(asctime)s %(levelname)s %(filename)s:%(lineno)s %(message)s",
-        level=arguments.log_level.upper(),
-    )
-
     if arguments.dev_mode:
-        print("Running rascsi-web in development mode ...")
+        logging.info("Dev mode enabled")
         APP.debug = True
         from werkzeug.debug import DebuggedApplication
 
@@ -1381,5 +1418,4 @@ if __name__ == "__main__":
         except KeyboardInterrupt:
             pass
     else:
-        print("Serving rascsi-web...")
         bjoern.run(APP, "0.0.0.0", arguments.port)

python/web/src/web_utils.py

@@ -324,10 +324,17 @@ def browser_supports_modern_themes():
     ]
 
     current_ua_family = user_agent["user_agent"]["family"]
-    current_ua_version = float(user_agent["user_agent"]["major"])
+    current_ua_version = user_agent["user_agent"]["major"]
     logging.info(f"Identified browser as family={current_ua_family}, version={current_ua_version}")
 
+    # Supported browsers cannot be identified without a version
+    if not current_ua_version:
+        return False
+
     for supported_browser, supported_version in supported_browsers:
-        if current_ua_family == supported_browser and current_ua_version >= supported_version:
+        if (
+            current_ua_family == supported_browser
+            and float(current_ua_version) >= supported_version
+        ):
             return True
     return False

python/web/start.sh

@@ -121,13 +121,14 @@ while [ "$1" != "" ]; do
 done
 
 PYTHON_COMMON_PATH=$(dirname $PWD)/common/src
-echo "Starting web server for RaSCSI Web Interface..."
 export PYTHONPATH=$PWD/src:${PYTHON_COMMON_PATH}
 cd src
 
 if [[ $ARG_DEV_MODE ]]; then
+    echo "Starting web UI (dev mode) ..."
     watchmedo auto-restart --directory=../../ --pattern=*.py --recursive -- \
     python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL} ${ARG_DEV_MODE}
 else
+    echo "Starting web UI ..."
     python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL} ${ARG_DEV_MODE}
 fi

python/web/tests/api/conftest.py

@@ -1,6 +1,7 @@
 import pytest
 import uuid
 import warnings
+import datetime
 
 SCSI_ID = 6
 FILE_SIZE_1_MIB = 1048576
@@ -13,7 +14,7 @@ def create_test_image(request, http_client):
     images = []
 
     def create(image_type="hds", size=1, auto_delete=True):
-        file_prefix = str(uuid.uuid4())
+        file_prefix = f"{request.function.__name__}___{uuid.uuid4()}"
         file_name = f"{file_prefix}.{image_type}"
 
         response = http_client.post(
@@ -29,13 +30,19 @@ def create_test_image(request, http_client):
             raise Exception("Failed to create temporary image")
 
         if auto_delete:
-            images.append(file_name)
+            images.append(
+                {
+                    "file_name": file_name,
+                    "function": request.function,
+                    "created": str(datetime.datetime.now()),
+                }
+            )
 
         return file_name
 
     def delete():
         for image in images:
-            response = http_client.post("/files/delete", data={"file_name": image})
+            response = http_client.post("/files/delete", data={"file_name": image["file_name"]})
             if response.status_code != 200 or response.json()["status"] != STATUS_SUCCESS:
                 warnings.warn(
                     f"Failed to auto-delete file created with create_test_image fixture: {image}"

python/web/tests/api/test_misc.py

@@ -112,3 +112,9 @@ def test_show_manpage(http_client):
 
     assert response.status_code == 200
     assert "rascsi" in response_data["data"]["manpage"]
+
+
+# route("/healthcheck", methods=["GET"])
+def test_healthcheck(http_client):
+    response = http_client.get("/healthcheck")
+    assert response.status_code == 200

python/web/tests/conftest.py

@@ -5,7 +5,7 @@ import os
 
 
 def pytest_addoption(parser):
-    default_base_url = "http://rascsi_web" if os.getenv("DOCKER") else "http://localhost:8080"
+    default_base_url = "http://web" if os.getenv("DOCKER") else "http://localhost:8080"
 
     parser.addoption("--home_dir", action="store", default="/home/pi")
     parser.addoption("--base_url", action="store", default=default_base_url)