diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ce6e2cda --- /dev/null +++ b/.dockerignore @@ -0,0 +1,32 @@ +# Exclude all by default +/* + +# Paths to include +!/docker/rascsi/rascsi_wrapper.sh +!/docker/rascsi/cfilesystem.patch +!/docker/rascsi-web/start.sh +!/doc +!/python +!/src +!/test +!/easyinstall.sh +!/LICENCE +!/lido-driver.img +!/README.md + +# From .gitignore +venv +*.pyc +core +**/.idea +.DS_Store +*.swp +__pycache__ +current +rascsi_interface_pb2.py +src/raspberrypi/hfdisk/ +*~ +messages.pot +messages.mo +s.sh +*-backups diff --git a/.gitignore b/.gitignore index f407b1a6..7a14233f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,9 +12,14 @@ src/raspberrypi/hfdisk/ messages.pot messages.mo +docker/docker-compose.override.yml +/docker/volumes/images/* +!/docker/volumes/images/.gitkeep +/docker/volumes/config/* +!/docker/volumes/config/.gitkeep + # temporary user files s.sh # temporary kicad files *-backups - diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..a99fd699 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,115 @@ +# Docker Environment for Development and Testing + +⚠️ **Important:** The Docker environment is unable to connect to the RaSCSI board and is +intended for development and testing purposes only. To setup RaSCSI on a Raspberry Pi +refer to the [setup instructions](https://github.com/akuker/RASCSI/wiki/Setup-Instructions) +on the wiki instead. + +## Introduction + +This documentation currently focuses on using Docker for developing and testing the web UI. + +Additions, amendments and contributions for additional workflows are most welcome. + +## Getting Started + +The easiest way to launch a new environment is to use Docker Compose. + +``` +cd docker +docker compose up +``` + +Containers will be built and started for the RaSCSI server and the web UI. + +The web UI can be accessed at: + +* http://localhost:8080 +* https://localhost:8443 + +To stop the containers, press *Ctrl + C*, or run `docker compose stop` +from another terminal. + +## Environment Variables + +The following environment variables are available when using Docker Compose: + +| Environment Variable | Default | +| -------------------- | -------- | +| `OS_DISTRO` | debian | +| `OS_VERSION` | buster | +| `OS_ARCH` | amd64 | +| `WEB_HTTP_PORT` | 8080 | +| `WEB_HTTPS_PORT` | 8443 | +| `WEB_LOG_LEVEL` | info | +| `RASCSI_HOST` | rascsi | +| `RASCSI_PORT` | 6868 | +| `RASCSI_PASSWORD` | *[None]* | +| `RASCSI_LOG_LEVEL` | debug | + +**Examples:** + +Run Debian "bullseye": +``` +OS_VERSION=bullseye docker compose up +``` + +Start the web UI with the log level set to debug: +``` +WEB_LOG_LEVEL=debug docker compose up +``` + +## Volumes + +When using Docker Compose the following volumes will be mounted automatically: + +| Local Path | Container Path | +| ----------------------- | ------------------------ | +| docker/volumes/images/ | /home/pi/images/ | +| docker/volumes/config/ | /home/pi/.config/rascsi/ | + + +## How To + +### Rebuild Containers + +You should rebuild the container images after checking out a different version of +RaSCSI or making changes which affect the environment at build time, e.g. +`easyinstall.sh`. + +``` +docker compose up --build +``` + +### Open a Shell on a Running Container + +Run the following command, replacing `[CONTAINER]` with `rascsi` or `rascsi_web`. + +``` +docker compose exec [CONTAINER] bash +``` + +### Setup Live Editing for the Web UI + +Use a `docker-compose.override.yml` to mount the local `python` directory to +`/home/pi/RASCSI/python/` in the `rascsi_web` container. + +Any changes to *.py files on the host computer (i.e. in your IDE) will trigger +the web UI process to be restarted in the container. + +**Example:** +``` +services: + rascsi_web: + volumes: + - ../python:/home/pi/RASCSI/python:delegated +``` + +### Connect the Web UI to a Real RaSCSI + +This can be useful for testing, but there are some caveats, e.g. the RaSCSI and the +web UI will be accessing separate `images` directories. + +``` +RASCSI_HOST=foo RASCSI_PASSWORD=bar docker compose up +``` \ No newline at end of file diff --git a/docker/docker-compose.override.yml.example b/docker/docker-compose.override.yml.example new file mode 100644 index 00000000..725fdda5 --- /dev/null +++ b/docker/docker-compose.override.yml.example @@ -0,0 +1,4 @@ +services: + rascsi_web: + volumes: + - ../python:/home/pi/RASCSI/python:delegated diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..4f150688 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,57 @@ +services: + rascsi: + container_name: rascsi + image: rascsi:develop-${OS_DISTRO:-debian}-${OS_VERSION:-buster}-${OS_ARCH:-amd64}-standalone + pull_policy: never + build: + context: .. + dockerfile: docker/rascsi/Dockerfile + args: + - OS_DISTRO=${OS_DISTRO:-debian} + - OS_VERSION=${OS_VERSION:-buster} + - OS_ARCH=${OS_ARCH:-amd64} + volumes: + - ./volumes/images:/home/pi/images:delegated + - ./volumes/config:/home/pi/.config/rascsi:delegated + ports: + - "127.0.0.1:${RASCSI_PORT:-6868}:6868" + environment: + - RASCSI_PASSWORD=${RASCSI_PASSWORD:-} + init: true + command: [ + "/usr/local/bin/rascsi_wrapper.sh", + "-L", + "${RASCSI_LOG_LEVEL:-trace}", + "-r", + "7", + "-F", + "/home/pi/images" + ] + + rascsi_web: + container_name: rascsi_web + image: rascsi:develop-${OS_DISTRO:-debian}-${OS_VERSION:-buster}-${OS_ARCH:-amd64}-web + pull_policy: never + build: + context: .. + dockerfile: docker/rascsi-web/Dockerfile + args: + - OS_DISTRO=${OS_DISTRO:-debian} + - OS_VERSION=${OS_VERSION:-buster} + - OS_ARCH=${OS_ARCH:-amd64} + volumes: + - ./volumes/images:/home/pi/images:delegated + - ./volumes/config:/home/pi/.config/rascsi:delegated + ports: + - "127.0.0.1:${WEB_HTTP_PORT:-8080}:80" + - "127.0.0.1:${WEB_HTTPS_PORT:-8443}:443" + environment: + - RASCSI_PASSWORD=${RASCSI_PASSWORD:-} + init: true + command: [ + "start.sh", + "--rascsi-host=${RASCSI_HOST:-rascsi}", + "--rascsi-port=${RASCSI_PORT:-6868}", + "--log-level=${WEB_LOG_LEVEL:-info}", + "--dev-mode" + ] diff --git a/docker/rascsi-web/Dockerfile b/docker/rascsi-web/Dockerfile new file mode 100644 index 00000000..4c4298d5 --- /dev/null +++ b/docker/rascsi-web/Dockerfile @@ -0,0 +1,24 @@ +ARG OS_DISTRO=debian +ARG OS_VERSION=buster +ARG OS_ARCH=amd64 +FROM "${OS_ARCH}/${OS_DISTRO}:${OS_VERSION}" + +EXPOSE 80 443 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends sudo rsyslog procps + +RUN groupadd pi +RUN useradd --create-home --shell /bin/bash -g pi pi +RUN echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +WORKDIR /home/pi +USER pi +COPY --chown=pi:pi . RASCSI +RUN cd RASCSI && ./easyinstall.sh --run_choice=11 --skip-token + +USER root +RUN pip3 install watchdog +COPY docker/rascsi-web/start.sh /usr/local/bin/start.sh +RUN chmod +x /usr/local/bin/start.sh +CMD ["/usr/local/bin/start.sh"] diff --git a/docker/rascsi-web/start.sh b/docker/rascsi-web/start.sh new file mode 100644 index 00000000..c282b6d0 --- /dev/null +++ b/docker/rascsi-web/start.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +if ! [[ -f "/home/pi/RASCSI/python/common/src/rascsi_interface_pb2.py" ]]; then + # Build rascsi_interface_pb2.py with the protobuf compiler + protoc \ + -I=/home/pi/RASCSI/src/raspberrypi \ + --python_out=/home/pi/RASCSI/python/common/src \ + rascsi_interface.proto +fi + +# Start Nginx service +nginx + +# Pass args to web UI start script +if [[ $RASCSI_PASSWORD ]]; then + /home/pi/RASCSI/python/web/start.sh "$@" --password=$RASCSI_PASSWORD +else + /home/pi/RASCSI/python/web/start.sh "$@" +fi diff --git a/docker/rascsi/Dockerfile b/docker/rascsi/Dockerfile new file mode 100644 index 00000000..508bd185 --- /dev/null +++ b/docker/rascsi/Dockerfile @@ -0,0 +1,29 @@ +ARG OS_DISTRO=debian +ARG OS_VERSION=buster +ARG OS_ARCH=amd64 +FROM "${OS_ARCH}/${OS_DISTRO}:${OS_VERSION}" + +EXPOSE 6868 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends sudo rsyslog patch + +RUN groupadd pi +RUN useradd --create-home --shell /bin/bash -g pi pi +RUN echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +USER pi +COPY --chown=pi:pi . /home/pi/RASCSI +WORKDIR /home/pi/RASCSI + +# Workaround for Bullseye amd64 compilation error +# https://github.com/akuker/RASCSI/issues/821 +RUN patch -p0 < docker/rascsi/cfilesystem.patch + +# Install RaSCSI standalone +RUN ./easyinstall.sh --run_choice=10 --cores=`nproc` --skip-token + +USER root +COPY docker/rascsi/rascsi_wrapper.sh /usr/local/bin/rascsi_wrapper.sh +RUN chmod +x /usr/local/bin/rascsi_wrapper.sh +CMD ["/usr/local/bin/rascsi_wrapper.sh", "-L", "trace", "-r", "7", "-F", "/home/pi/images"] diff --git a/docker/rascsi/cfilesystem.patch b/docker/rascsi/cfilesystem.patch new file mode 100644 index 00000000..32d96e93 --- /dev/null +++ b/docker/rascsi/cfilesystem.patch @@ -0,0 +1,24 @@ +--- src/raspberrypi/devices/cfilesystem.cpp 2022-09-08 12:07:14.000000000 +0100 ++++ src/raspberrypi/devices/cfilesystem.cpp.patched 2022-09-08 12:12:55.000000000 +0100 +@@ -1075,12 +1075,15 @@ + m_dirHuman.name[i] = ' '; + } + +- for (i = 0; i < 10; i++) { +- if (p < m_pszHumanExt) +- m_dirHuman.add[i] = *p++; +- else +- m_dirHuman.add[i] = '\0'; +- } ++ // This code causes a compilation error on Debian "bullseye" (amd64) ++ // https://github.com/akuker/RASCSI/issues/821 ++ // ++ // for (i = 0; i < 10; i++) { ++ // if (p < m_pszHumanExt) ++ // m_dirHuman.add[i] = *p++; ++ // else ++ // m_dirHuman.add[i] = '\0'; ++ // } + + if (*p == '.') + p++; diff --git a/docker/rascsi/rascsi_wrapper.sh b/docker/rascsi/rascsi_wrapper.sh new file mode 100644 index 00000000..4e750ac3 --- /dev/null +++ b/docker/rascsi/rascsi_wrapper.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +if [[ $RASCSI_PASSWORD ]]; then + TOKEN_FILE="/home/pi/.config/rascsi/rascsi_secret" + mkdir -p /home/pi/.config/rascsi || true + echo $RASCSI_PASSWORD > $TOKEN_FILE + chmod 700 $TOKEN_FILE + /usr/local/bin/rascsi "$@" -P $TOKEN_FILE +else + /usr/local/bin/rascsi "$@" +fi diff --git a/docker/volumes/config/.gitkeep b/docker/volumes/config/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docker/volumes/images/.gitkeep b/docker/volumes/images/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/easyinstall.sh b/easyinstall.sh index f44c6fc5..4d4fcc7a 100755 --- a/easyinstall.sh +++ b/easyinstall.sh @@ -63,6 +63,7 @@ LIDO_DRIVER=$BASE/lido-driver.img GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) GIT_REMOTE=${GIT_REMOTE:-origin} TOKEN="" +SECRET_FILE="$HOME/.config/rascsi/rascsi_secret" set -e @@ -82,10 +83,40 @@ function sudoCheck() { # install all dependency packages for RaSCSI Service function installPackages() { - sudo apt-get update && sudo apt-get install git libspdlog-dev libpcap-dev \ - genisoimage python3 python3-venv python3-dev python3-pip nginx \ - libpcap-dev protobuf-compiler bridge-utils libev-dev libevdev2 unar \ - disktype libgmock-dev -y "$SECRET_FILE" + fi + + echo "$TOKEN" > "$SECRET_FILE" # Make the secret file owned and only readable by root - sudo chown root:root "$SECRET_FILE" - sudo chmod 600 "$SECRET_FILE" - echo "" - echo "Configured RaSCSI to use $SECRET_FILE for authentication. This file is readable by root only." - echo "Make note of your password: you will need it to use rasctl and other RaSCSI clients." - fi + sudo chown root:root "$SECRET_FILE" + sudo chmod 600 "$SECRET_FILE" + echo "" + echo "Configured RaSCSI to use $SECRET_FILE for authentication. This file is readable by root only." + echo "Make note of your password: you will need it to use rasctl and other RaSCSI clients." } # Modifies and installs the rascsi service @@ -1202,8 +1243,9 @@ function runChoice() { echo "- Install manpages to /usr/local/man" sudoCheck createImagesDir + configureTokenAuth updateRaScsiGit - installPackages + installPackagesStandalone stopRaScsi compileRaScsi installRaScsi @@ -1221,6 +1263,7 @@ function runChoice() { echo "- Create a self-signed certificate in /etc/ssl" sudoCheck createCfgDir + configureTokenAuth updateRaScsiGit installPackages preparePythonCommon @@ -1294,24 +1337,38 @@ while [ "$1" != "" ]; do VALUE=$(echo "$1" | awk -F= '{print $2}') case $PARAM in -c | --connect_type) + if ! [[ $VALUE =~ ^(FULLSPEC|STANDARD|AIBOM|GAMERNIUM)$ ]]; then + echo "ERROR: The connect type parameter must have a value of: FULLSPEC, STANDARD, AIBOM or GAMERNIUM" + exit 1 + fi CONNECT_TYPE=$VALUE ;; -r | --run_choice) + if ! [[ $VALUE =~ ^[1-9][0-9]?$ && $VALUE -ge 1 && $VALUE -le 12 ]]; then + echo "ERROR: The run choice parameter must have a numeric value between 1 and 12" + exit 1 + fi RUN_CHOICE=$VALUE ;; -j | --cores) + if ! [[ $VALUE =~ ^[1-9][0-9]?$ ]]; then + echo "ERROR: The cores parameter must have a numeric value of at least 1" + exit 1 + fi CORES=$VALUE ;; - *) - echo "ERROR: unknown parameter \"$PARAM\"" - exit 1 + -t | --token) + if [[ -z $VALUE ]]; then + echo "ERROR: The token parameter cannot be empty" + exit 1 + fi + TOKEN=$VALUE ;; - esac - case $VALUE in - FULLSPEC | STANDARD | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12) + -s | --skip-token) + SKIP_TOKEN=1 ;; *) - echo "ERROR: unknown option \"$VALUE\"" + echo "ERROR: Unknown parameter \"$PARAM\"" exit 1 ;; esac diff --git a/python/common/src/util/unarchiver.py b/python/common/src/util/unarchiver.py index 7bbb4b9a..d9cfd61d 100644 --- a/python/common/src/util/unarchiver.py +++ b/python/common/src/util/unarchiver.py @@ -10,6 +10,7 @@ import pathlib from tempfile import TemporaryDirectory from re import escape, match from json import loads, JSONDecodeError +from shutil import move from util.run import run FORK_OUTPUT_TYPE_VISIBLE = "visible" @@ -140,7 +141,7 @@ def extract_archive(file_path, **kwargs): # The parent dir may not be specified as a member, so ensure it exists target_path.parent.mkdir(parents=True, exist_ok=True) logging.debug("Moving temp file: %s -> %s", source_path, target_path) - source_path.rename(target_path) + move(source_path, target_path) moved.append(member) return { diff --git a/python/web/requirements.txt b/python/web/requirements.txt index c9ced2c3..5823fd45 100644 --- a/python/web/requirements.txt +++ b/python/web/requirements.txt @@ -4,7 +4,7 @@ Flask==2.0.1 itsdangerous==2.0.1 Jinja2==3.0.1 MarkupSafe==2.0.1 -protobuf==3.17.3 +protobuf==3.20.1 requests==2.26.0 simplepam==0.1.5 flask_babel==2.0.0 diff --git a/python/web/src/web.py b/python/web/src/web.py index 22e1084b..a51f308f 100644 --- a/python/web/src/web.py +++ b/python/web/src/web.py @@ -1048,6 +1048,11 @@ if __name__ == "__main__": help="Log level for Web UI. Default: warning", choices=["debug", "info", "warning", "error", "critical"], ) + parser.add_argument( + "--dev-mode", + action="store_true", + help="Run in development mode" + ) arguments = parser.parse_args() APP.config["TOKEN"] = arguments.password @@ -1064,5 +1069,14 @@ if __name__ == "__main__": format="%(asctime)s %(levelname)s %(filename)s:%(lineno)s %(message)s", level=arguments.log_level.upper()) - print("Serving rascsi-web...") - bjoern.run(APP, "0.0.0.0", arguments.port) + if arguments.dev_mode: + print("Running rascsi-web in development mode ...") + APP.debug = True + from werkzeug.debug import DebuggedApplication + try: + bjoern.run(DebuggedApplication(APP, evalex=False), "0.0.0.0", arguments.port) + except KeyboardInterrupt: + pass + else: + print("Serving rascsi-web...") + bjoern.run(APP, "0.0.0.0", arguments.port) diff --git a/python/web/start.sh b/python/web/start.sh index 7954c0ea..f9e5b26b 100755 --- a/python/web/start.sh +++ b/python/web/start.sh @@ -110,6 +110,9 @@ while [ "$1" != "" ]; do -l | --log-level) ARG_LOG_LEVEL="--log-level $VALUE" ;; + -d | --dev-mode) + ARG_DEV_MODE="--dev-mode" + ;; *) echo "ERROR: unknown parameter \"$PARAM\"" exit 1 @@ -122,4 +125,10 @@ PYTHON_COMMON_PATH=$(dirname $PWD)/common/src echo "Starting web server for RaSCSI Web Interface..." export PYTHONPATH=$PWD/src:${PYTHON_COMMON_PATH} cd src -python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL} + +if [[ $ARG_DEV_MODE ]]; then + watchmedo auto-restart --directory=../../ --pattern=*.py --recursive -- \ + python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL} ${ARG_DEV_MODE} +else + python3 web.py ${ARG_PORT} ${ARG_PASSWORD} ${ARG_RASCSI_HOST} ${ARG_RASCSI_PORT} ${ARG_LOG_LEVEL} ${ARG_DEV_MODE} +fi \ No newline at end of file