From 673da6312ba77d9a636ba4d37ef3e66a14713601 Mon Sep 17 00:00:00 2001 From: nucleogenic Date: Sun, 28 Aug 2022 14:51:31 +0100 Subject: [PATCH] Add Docker environment for development and testing of the web UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add --token parameter to easyinstall.sh Add --skip-token parameter to easyinstall.sh Install required apt packages explicitly (--no-install-recommends) Allow standalone RaSCSI and web UI installations to specify an auth token Add development mode to web UI (web/start.sh --dev-mode) Initial Docker-based development environment for Python and web UI Bump protobuf version Workaround for Flask development server and asyncio incompatibility Build Python protobuf interface on container launch, if it doesn’t exist Allow containers to be configured with environment variables, add support for token authentication Move web UI live editing setup out of main Docker Compose config Update dockerignore to exclude by default Update README Add OS_DISTRO, OS_VERSION and OS_ARCH build args Allow extracted files to be moved to target when crossing a filesystem boundary Reduce noise from watchmedo auto-restarts Update Docker tag structure to rascsi:{build}-{platform}-{variant} Prevent Docker Compose from attempting to pull images from Docker registry Add workaround for issue #821 Allow container processes to be stopped with Ctrl+C Update README, bind to ports 8080/8443 on the Docker host by default Update README to clarify audience and no board connectivity Add AIBOM and GAMERNIUM to --connect_type validation Update cfilesystem.patch following rebase --- .dockerignore | 32 ++++++ .gitignore | 7 +- docker/README.md | 115 ++++++++++++++++++++ docker/docker-compose.override.yml.example | 4 + docker/docker-compose.yml | 57 ++++++++++ docker/rascsi-web/Dockerfile | 24 ++++ docker/rascsi-web/start.sh | 19 ++++ docker/rascsi/Dockerfile | 29 +++++ docker/rascsi/cfilesystem.patch | 24 ++++ docker/rascsi/rascsi_wrapper.sh | 11 ++ docker/volumes/config/.gitkeep | 0 docker/volumes/images/.gitkeep | 0 easyinstall.sh | 121 +++++++++++++++------ python/common/src/util/unarchiver.py | 3 +- python/web/requirements.txt | 2 +- python/web/src/web.py | 18 ++- python/web/start.sh | 11 +- 17 files changed, 439 insertions(+), 38 deletions(-) create mode 100644 .dockerignore create mode 100644 docker/README.md create mode 100644 docker/docker-compose.override.yml.example create mode 100644 docker/docker-compose.yml create mode 100644 docker/rascsi-web/Dockerfile create mode 100644 docker/rascsi-web/start.sh create mode 100644 docker/rascsi/Dockerfile create mode 100644 docker/rascsi/cfilesystem.patch create mode 100644 docker/rascsi/rascsi_wrapper.sh create mode 100644 docker/volumes/config/.gitkeep create mode 100644 docker/volumes/images/.gitkeep diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ce6e2cda --- /dev/null +++ b/.dockerignore @@ -0,0 +1,32 @@ +# Exclude all by default +/* + +# Paths to include +!/docker/rascsi/rascsi_wrapper.sh +!/docker/rascsi/cfilesystem.patch +!/docker/rascsi-web/start.sh +!/doc +!/python +!/src +!/test +!/easyinstall.sh +!/LICENCE +!/lido-driver.img +!/README.md + +# From .gitignore +venv +*.pyc +core +**/.idea +.DS_Store +*.swp +__pycache__ +current +rascsi_interface_pb2.py +src/raspberrypi/hfdisk/ +*~ +messages.pot +messages.mo +s.sh +*-backups diff --git a/.gitignore b/.gitignore index f407b1a6..7a14233f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,9 +12,14 @@ src/raspberrypi/hfdisk/ messages.pot messages.mo +docker/docker-compose.override.yml +/docker/volumes/images/* +!/docker/volumes/images/.gitkeep +/docker/volumes/config/* +!/docker/volumes/config/.gitkeep + # temporary user files s.sh # temporary kicad files *-backups - diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..a99fd699 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,115 @@ +# Docker Environment for Development and Testing + +⚠️ **Important:** The Docker environment is unable to connect to the RaSCSI board and is +intended for development and testing purposes only. To setup RaSCSI on a Raspberry Pi +refer to the [setup instructions](https://github.com/akuker/RASCSI/wiki/Setup-Instructions) +on the wiki instead. + +## Introduction + +This documentation currently focuses on using Docker for developing and testing the web UI. + +Additions, amendments and contributions for additional workflows are most welcome. + +## Getting Started + +The easiest way to launch a new environment is to use Docker Compose. + +``` +cd docker +docker compose up +``` + +Containers will be built and started for the RaSCSI server and the web UI. + +The web UI can be accessed at: + +* http://localhost:8080 +* https://localhost:8443 + +To stop the containers, press *Ctrl + C*, or run `docker compose stop` +from another terminal. + +## Environment Variables + +The following environment variables are available when using Docker Compose: + +| Environment Variable | Default | +| -------------------- | -------- | +| `OS_DISTRO` | debian | +| `OS_VERSION` | buster | +| `OS_ARCH` | amd64 | +| `WEB_HTTP_PORT` | 8080 | +| `WEB_HTTPS_PORT` | 8443 | +| `WEB_LOG_LEVEL` | info | +| `RASCSI_HOST` | rascsi | +| `RASCSI_PORT` | 6868 | +| `RASCSI_PASSWORD` | *[None]* | +| `RASCSI_LOG_LEVEL` | debug | + +**Examples:** + +Run Debian "bullseye": +``` +OS_VERSION=bullseye docker compose up +``` + +Start the web UI with the log level set to debug: +``` +WEB_LOG_LEVEL=debug docker compose up +``` + +## Volumes + +When using Docker Compose the following volumes will be mounted automatically: + +| Local Path | Container Path | +| ----------------------- | ------------------------ | +| docker/volumes/images/ | /home/pi/images/ | +| docker/volumes/config/ | /home/pi/.config/rascsi/ | + + +## How To + +### Rebuild Containers + +You should rebuild the container images after checking out a different version of +RaSCSI or making changes which affect the environment at build time, e.g. +`easyinstall.sh`. + +``` +docker compose up --build +``` + +### Open a Shell on a Running Container + +Run the following command, replacing `[CONTAINER]` with `rascsi` or `rascsi_web`. + +``` +docker compose exec [CONTAINER] bash +``` + +### Setup Live Editing for the Web UI + +Use a `docker-compose.override.yml` to mount the local `python` directory to +`/home/pi/RASCSI/python/` in the `rascsi_web` container. + +Any changes to *.py files on the host computer (i.e. in your IDE) will trigger +the web UI process to be restarted in the container. + +**Example:** +``` +services: + rascsi_web: + volumes: + - ../python:/home/pi/RASCSI/python:delegated +``` + +### Connect the Web UI to a Real RaSCSI + +This can be useful for testing, but there are some caveats, e.g. the RaSCSI and the +web UI will be accessing separate `images` directories. + +``` +RASCSI_HOST=foo RASCSI_PASSWORD=bar docker compose up +``` \ No newline at end of file diff --git a/docker/docker-compose.override.yml.example b/docker/docker-compose.override.yml.example new file mode 100644 index 00000000..725fdda5 --- /dev/null +++ b/docker/docker-compose.override.yml.example @@ -0,0 +1,4 @@ +services: + rascsi_web: + volumes: + - ../python:/home/pi/RASCSI/python:delegated diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..4f150688 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,57 @@ +services: + rascsi: + container_name: rascsi + image: rascsi:develop-${OS_DISTRO:-debian}-${OS_VERSION:-buster}-${OS_ARCH:-amd64}-standalone + pull_policy: never + build: + context: .. + dockerfile: docker/rascsi/Dockerfile + args: + - OS_DISTRO=${OS_DISTRO:-debian} + - OS_VERSION=${OS_VERSION:-buster} + - OS_ARCH=${OS_ARCH:-amd64} + volumes: + - ./volumes/images:/home/pi/images:delegated + - ./volumes/config:/home/pi/.config/rascsi:delegated + ports: + - "127.0.0.1:${RASCSI_PORT:-6868}:6868" + environment: + - RASCSI_PASSWORD=${RASCSI_PASSWORD:-} + init: true + command: [ + "/usr/local/bin/rascsi_wrapper.sh", + "-L", + "${RASCSI_LOG_LEVEL:-trace}", + "-r", + "7", + "-F", + "/home/pi/images" + ] + + rascsi_web: + container_name: rascsi_web + image: rascsi:develop-${OS_DISTRO:-debian}-${OS_VERSION:-buster}-${OS_ARCH:-amd64}-web + pull_policy: never + build: + context: .. + dockerfile: docker/rascsi-web/Dockerfile + args: + - OS_DISTRO=${OS_DISTRO:-debian} + - OS_VERSION=${OS_VERSION:-buster} + - OS_ARCH=${OS_ARCH:-amd64} + volumes: + - ./volumes/images:/home/pi/images:delegated + - ./volumes/config:/home/pi/.config/rascsi:delegated + ports: + - "127.0.0.1:${WEB_HTTP_PORT:-8080}:80" + - "127.0.0.1:${WEB_HTTPS_PORT:-8443}:443" + environment: + - RASCSI_PASSWORD=${RASCSI_PASSWORD:-} + init: true + command: [ + "start.sh", + "--rascsi-host=${RASCSI_HOST:-rascsi}", + "--rascsi-port=${RASCSI_PORT:-6868}", + "--log-level=${WEB_LOG_LEVEL:-info}", + "--dev-mode" + ] diff --git a/docker/rascsi-web/Dockerfile b/docker/rascsi-web/Dockerfile new file mode 100644 index 00000000..4c4298d5 --- /dev/null +++ b/docker/rascsi-web/Dockerfile @@ -0,0 +1,24 @@ +ARG OS_DISTRO=debian +ARG OS_VERSION=buster +ARG OS_ARCH=amd64 +FROM "${OS_ARCH}/${OS_DISTRO}:${OS_VERSION}" + +EXPOSE 80 443 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends sudo rsyslog procps + +RUN groupadd pi +RUN useradd --create-home --shell /bin/bash -g pi pi +RUN echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +WORKDIR /home/pi +USER pi +COPY --chown=pi:pi . RASCSI +RUN cd RASCSI && ./easyinstall.sh --run_choice=11 --skip-token + +USER root +RUN pip3 install watchdog +COPY docker/rascsi-web/start.sh /usr/local/bin/start.sh +RUN chmod +x /usr/local/bin/start.sh +CMD ["/usr/local/bin/start.sh"] diff --git a/docker/rascsi-web/start.sh b/docker/rascsi-web/start.sh new file mode 100644 index 00000000..c282b6d0 --- /dev/null +++ b/docker/rascsi-web/start.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +if ! [[ -f "/home/pi/RASCSI/python/common/src/rascsi_interface_pb2.py" ]]; then + # Build rascsi_interface_pb2.py with the protobuf compiler + protoc \ + -I=/home/pi/RASCSI/src/raspberrypi \ + --python_out=/home/pi/RASCSI/python/common/src \ + rascsi_interface.proto +fi + +# Start Nginx service +nginx + +# Pass args to web UI start script +if [[ $RASCSI_PASSWORD ]]; then + /home/pi/RASCSI/python/web/start.sh "$@" --password=$RASCSI_PASSWORD +else + /home/pi/RASCSI/python/web/start.sh "$@" +fi diff --git a/docker/rascsi/Dockerfile b/docker/rascsi/Dockerfile new file mode 100644 index 00000000..508bd185 --- /dev/null +++ b/docker/rascsi/Dockerfile @@ -0,0 +1,29 @@ +ARG OS_DISTRO=debian +ARG OS_VERSION=buster +ARG OS_ARCH=amd64 +FROM "${OS_ARCH}/${OS_DISTRO}:${OS_VERSION}" + +EXPOSE 6868 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends sudo rsyslog patch + +RUN groupadd pi +RUN useradd --create-home --shell /bin/bash -g pi pi +RUN echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +USER pi +COPY --chown=pi:pi . /home/pi/RASCSI +WORKDIR /home/pi/RASCSI + +# Workaround for Bullseye amd64 compilation error +# https://github.com/akuker/RASCSI/issues/821 +RUN patch -p0 < docker/rascsi/cfilesystem.patch + +# Install RaSCSI standalone +RUN ./easyinstall.sh --run_choice=10 --cores=`nproc` --skip-token + +USER root +COPY docker/rascsi/rascsi_wrapper.sh /usr/local/bin/rascsi_wrapper.sh +RUN chmod +x /usr/local/bin/rascsi_wrapper.sh +CMD ["/usr/local/bin/rascsi_wrapper.sh", "-L", "trace", "-r", "7", "-F", "/home/pi/images"] diff --git a/docker/rascsi/cfilesystem.patch b/docker/rascsi/cfilesystem.patch new file mode 100644 index 00000000..32d96e93 --- /dev/null +++ b/docker/rascsi/cfilesystem.patch @@ -0,0 +1,24 @@ +--- src/raspberrypi/devices/cfilesystem.cpp 2022-09-08 12:07:14.000000000 +0100 ++++ src/raspberrypi/devices/cfilesystem.cpp.patched 2022-09-08 12:12:55.000000000 +0100 +@@ -1075,12 +1075,15 @@ + m_dirHuman.name[i] = ' '; + } + +- for (i = 0; i < 10; i++) { +- if (p < m_pszHumanExt) +- m_dirHuman.add[i] = *p++; +- else +- m_dirHuman.add[i] = '\0'; +- } ++ // This code causes a compilation error on Debian "bullseye" (amd64) ++ // https://github.com/akuker/RASCSI/issues/821 ++ // ++ // for (i = 0; i < 10; i++) { ++ // if (p < m_pszHumanExt) ++ // m_dirHuman.add[i] = *p++; ++ // else ++ // m_dirHuman.add[i] = '\0'; ++ // } + + if (*p == '.') + p++; diff --git a/docker/rascsi/rascsi_wrapper.sh b/docker/rascsi/rascsi_wrapper.sh new file mode 100644 index 00000000..4e750ac3 --- /dev/null +++ b/docker/rascsi/rascsi_wrapper.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +if [[ $RASCSI_PASSWORD ]]; then + TOKEN_FILE="/home/pi/.config/rascsi/rascsi_secret" + mkdir -p /home/pi/.config/rascsi || true + echo $RASCSI_PASSWORD > $TOKEN_FILE + chmod 700 $TOKEN_FILE + /usr/local/bin/rascsi "$@" -P $TOKEN_FILE +else + /usr/local/bin/rascsi "$@" +fi diff --git a/docker/volumes/config/.gitkeep b/docker/volumes/config/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docker/volumes/images/.gitkeep b/docker/volumes/images/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/easyinstall.sh b/easyinstall.sh index f44c6fc5..4d4fcc7a 100755 --- a/easyinstall.sh +++ b/easyinstall.sh @@ -63,6 +63,7 @@ LIDO_DRIVER=$BASE/lido-driver.img GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) GIT_REMOTE=${GIT_REMOTE:-origin} TOKEN="" +SECRET_FILE="$HOME/.config/rascsi/rascsi_secret" set -e @@ -82,10 +83,40 @@ function sudoCheck() { # install all dependency packages for RaSCSI Service function installPackages() { - sudo apt-get update && sudo apt-get install git libspdlog-dev libpcap-dev \ - genisoimage python3 python3-venv python3-dev python3-pip nginx \ - libpcap-dev protobuf-compiler bridge-utils libev-dev libevdev2 unar \ - disktype libgmock-dev -y "$SECRET_FILE" + fi + + echo "$TOKEN" > "$SECRET_FILE" # Make the secret file owned and only readable by root - sudo chown root:root "$SECRET_FILE" - sudo chmod 600 "$SECRET_FILE" - echo "" - echo "Configured RaSCSI to use $SECRET_FILE for authentication. This file is readable by root only." - echo "Make note of your password: you will need it to use rasctl and other RaSCSI clients." - fi + sudo chown root:root "$SECRET_FILE" + sudo chmod 600 "$SECRET_FILE" + echo "" + echo "Configured RaSCSI to use $SECRET_FILE for authentication. This file is readable by root only." + echo "Make note of your password: you will need it to use rasctl and other RaSCSI clients." } # Modifies and installs the rascsi service @@ -1202,8 +1243,9 @@ function runChoice() { echo "- Install manpages to /usr/local/man" sudoCheck createImagesDir + configureTokenAuth updateRaScsiGit - installPackages + installPackagesStandalone stopRaScsi compileRaScsi installRaScsi @@ -1221,6 +1263,7 @@ function runChoice() { echo "- Create a self-signed certificate in /etc/ssl" sudoCheck createCfgDir + configureTokenAuth updateRaScsiGit installPackages preparePythonCommon @@ -1294,24 +1337,38 @@ while [ "$1" != "" ]; do VALUE=$(echo "$1" | awk -F= '{print $2}') case $PARAM in -c | --connect_type) + if ! [[ $VALUE =~ ^(FULLSPEC|STANDARD|AIBOM|GAMERNIUM)$ ]]; then + echo "ERROR: The connect type parameter must have a value of: FULLSPEC, STANDARD, AIBOM or GAMERNIUM" + exit 1 + fi CONNECT_TYPE=$VALUE ;; -r | --run_choice) + if ! [[ $VALUE =~ ^[1-9][0-9]?$ && $VALUE -ge 1 && $VALUE -le 12 ]]; then + echo "ERROR: The run choice parameter must have a numeric value between 1 and 12" + exit 1 + fi RUN_CHOICE=$VALUE ;; -j | --cores) + if ! [[ $VALUE =~ ^[1-9][0-9]?$ ]]; then + echo "ERROR: The cores parameter must have a numeric value of at least 1" + exit 1 + fi CORES=$VALUE ;; - *) - echo "ERROR: unknown parameter \"$PARAM\"" - exit 1 + -t | --token) + if [[ -z $VALUE ]]; then + echo "ERROR: The token parameter cannot be empty" + exit 1 + fi + TOKEN=$VALUE ;; - esac - case $VALUE in - FULLSPEC | STANDARD | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12) + -s | --skip-token) + SKIP_TOKEN=1 ;; *) - echo "ERROR: unknown option \"$VALUE\"" + echo "ERROR: Unknown parameter \"$PARAM\"" exit 1 ;; esac diff --git a/python/common/src/util/unarchiver.py b/python/common/src/util/unarchiver.py index 7bbb4b9a..d9cfd61d 100644 --- a/python/common/src/util/unarchiver.py +++ b/python/common/src/util/unarchiver.py @@ -10,6 +10,7 @@ import pathlib from tempfile import TemporaryDirectory from re import escape, match from json import loads, JSONDecodeError +from shutil import move from util.run import run FORK_OUTPUT_TYPE_VISIBLE = "visible" @@ -140,7 +141,7 @@ def extract_archive(file_path, **kwargs): # The parent dir may not be specified as a member, so ensure it exists target_path.parent.mkdir(parents=True, exist_ok=True) logging.debug("Moving temp file: %s -> %s", source_path, target_path) - source_path.rename(target_path) + move(source_path, target_path) moved.append(member) return { diff --git a/python/web/requirements.txt b/python/web/requirements.txt index c9ced2c3..5823fd45 100644 --- a/python/web/requirements.txt +++ b/python/web/requirements.txt @@ -4,7 +4,7 @@ Flask==2.0.1 itsdangerous==2.0.1 Jinja2==3.0.1 MarkupSafe==2.0.1 -protobuf==3.17.3 +protobuf==3.20.1 requests==2.26.0 simplepam==0.1.5 flask_babel==2.0.0 diff --git a/python/web/src/web.py b/python/web/src/web.py index 22e1084b..a51f308f 100644 --- a/python/web/src/web.py +++ b/python/web/src/web.py @@ -1048,6 +1048,11 @@ if __name__ == "__main__": help="Log level for Web UI. Default: warning", choices=["debug", "info", "warning", "error", "critical"], ) + parser.add_argument( + "--dev-mode", + action="store_true", + help="Run in development mode" + ) arguments = parser.parse_args() APP.config["TOKEN"] = arguments.password @@ -1064,5 +1069,14 @@ if __name__ == "__main__": format="%(asctime)s %(levelname)s %(filename)s:%(lineno)s %(message)s", level=arguments.log_level.upper()) - print("Serving rascsi-web...") - bjoern.run(APP, "0.0.0.0", arguments.port) + if arguments.dev_mode: + print("Running rascsi-web in development mode ...") + APP.debug = True + from werkzeug.debug import DebuggedApplication + try: + bjoern.run(DebuggedApplication(APP, evalex=False), "0.0.0.0", arguments.port) + except KeyboardInterrupt: + pass + else: + print("Serving rascsi-web...") + bjoern.run(APP, "0.0.0.0", arguments.port) diff --git a/python/web/start.sh b/python/web/start.sh index 7954c0ea..f9e5b26b 100755 --- a/python/web/start.sh +++ b/python/web/start.sh @@ -110,6 +110,9 @@ while [ "$1" != "" ]; do -l | --log-level) ARG_LOG_LEVEL="--log-level $VALUE" ;; + -d | --dev-mode) + ARG_DEV_MODE="--dev-mode" + ;; *) echo "ERROR: unknown parameter \"$PARAM\"" exit 1 @@ -122,4 +125,10 @@ PYTHON_COMMON_PATH=$(dirname $PWD)/common/src echo "Starting web server for RaSCSI Web Interface..." export PYTHONPATH=$PWD/src:${PYTHON_COMMON_PATH} cd src -python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL} + +if [[ $ARG_DEV_MODE ]]; then + watchmedo auto-restart --directory=../../ --pattern=*.py --recursive -- \ + python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL} ${ARG_DEV_MODE} +else + python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL} ${ARG_DEV_MODE} +fi \ No newline at end of file