diff --git a/.dockerignore b/.dockerignore index 3f5010b3..2b3a49cd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,9 +2,8 @@ /* # Paths to include -!/docker/rascsi/rascsi_wrapper.sh -!/docker/rascsi/cfilesystem.patch -!/docker/rascsi-web/start.sh +!/docker/backend/rascsi_wrapper.sh +!/docker/web/start.sh !/doc !/python !/cpp @@ -13,19 +12,27 @@ !/LICENCE !/README.md -# From .gitignore -venv -*.pyc -core +# Dev artifacts to exclude +**/.git + +/cpp/bin +/cpp/obj + +**/venv* +**/*.pyc +**/__pycache__ +**/.pytest_cache +**/rascsi_interface_pb2.py +**/report.xml + **/.idea -.DS_Store -*.swp -__pycache__ -current -rascsi_interface_pb2.py -src/raspberrypi/hfdisk/ -*~ -messages.pot -messages.mo -s.sh -*-backups +**/.vscode +**/.DS_Store + +**/core +**/*.swp +**/current + +**/node_modules +**/messages.pot +**/messages.mo diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index da64b983..3809ede4 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -2,22 +2,18 @@ name: Web Tests/Analysis on: workflow_dispatch: - pull_request: - types: [opened, synchronize] + push: paths: - 'python/web/**' - 'python/common/**' - '.github/workflows/web.yml' - push: - branches: - - develop jobs: backend_checks: runs-on: ubuntu-latest defaults: run: - working-directory: python/web + working-directory: python steps: - uses: actions/checkout@v3 @@ -26,14 +22,88 @@ jobs: python-version: 3.7.15 cache: 'pip' - - run: pip install -r requirements-dev.txt + - run: pip install -r web/requirements-dev.txt id: pip - - run: black --check tests + - run: black --check . - - run: flake8 tests + - run: flake8 . if: success() || failure() && steps.pip.outcome == 'success' + backend_tests: + runs-on: ubuntu-latest + defaults: + run: + working-directory: docker + steps: + - uses: actions/checkout@v3 + + - name: Check DockerHub for existing backend image + run: | + export DOCKER_BACKEND_IMAGE="piscsi/backend-standalone:`git ls-files -s python .github/workflows/web.yml | git hash-object --stdin`" + echo "DOCKER_BACKEND_IMAGE=${DOCKER_BACKEND_IMAGE}" >> $GITHUB_ENV + docker pull --quiet ${DOCKER_BACKEND_IMAGE} || echo "DOCKER_BACKEND_NEEDS_PUSH=1" >> $GITHUB_ENV + + - name: Build and launch containers + run: docker compose -f docker-compose.ci.yml up -d + + - name: Run test suite + run: docker compose -f docker-compose.ci.yml run pytest -v + + - name: Check if DockerHub secrets defined + run: if [[ $DOCKERHUB_USERNAME && $DOCKERHUB_TOKEN ]]; then echo "DOCKERHUB_SECRETS_DEFINED=1" >> $GITHUB_ENV; fi + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to DockerHub + uses: docker/login-action@v2 + if: env.DOCKERHUB_SECRETS_DEFINED && env.DOCKER_BACKEND_NEEDS_PUSH + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push backend image to DockerHub + if: (success() || failure()) && env.DOCKERHUB_SECRETS_DEFINED && env.DOCKER_BACKEND_NEEDS_PUSH + run: docker compose -f docker-compose.ci.yml push backend + + - name: Upload test artifacts + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: pytest-output.zip + path: | + docker/volumes/pytest/report.xml + docker/volumes/pytest/pytest.log + + - name: Output container logs + if: success() || failure() + run: | + docker compose -f docker-compose.ci.yml logs backend > backend.log + docker compose -f docker-compose.ci.yml logs web > web.log + docker compose -f docker-compose.ci.yml logs -t | sort -u -k 3 > combined.log + + - name: Upload backend log + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: backend.log + path: docker/backend.log + + - name: Upload web log + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: web.log + path: docker/web.log + + - name: Upload combined log + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: combined.log + path: docker/combined.log + frontend_checks: runs-on: ubuntu-latest defaults: diff --git a/.gitignore b/.gitignore index 09aa2a39..69cd97f5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,15 @@ venv *.pyc +*.swp +*.log +*~ core .idea/ +.vscode .DS_Store -*.swp __pycache__ current rascsi_interface_pb2.py -*~ messages.pot messages.mo report.xml diff --git a/docker/README.md b/docker/README.md index a99fd699..02ba4d87 100644 --- a/docker/README.md +++ b/docker/README.md @@ -35,14 +35,12 @@ from another terminal. The following environment variables are available when using Docker Compose: | Environment Variable | Default | -| -------------------- | -------- | -| `OS_DISTRO` | debian | +| -------------------- |----------| | `OS_VERSION` | buster | -| `OS_ARCH` | amd64 | | `WEB_HTTP_PORT` | 8080 | | `WEB_HTTPS_PORT` | 8443 | | `WEB_LOG_LEVEL` | info | -| `RASCSI_HOST` | rascsi | +| `RASCSI_HOST` | backend | | `RASCSI_PORT` | 6868 | | `RASCSI_PASSWORD` | *[None]* | | `RASCSI_LOG_LEVEL` | debug | @@ -83,7 +81,7 @@ docker compose up --build ### Open a Shell on a Running Container -Run the following command, replacing `[CONTAINER]` with `rascsi` or `rascsi_web`. +Run the following command, replacing `[CONTAINER]` with `backend` or `web`. ``` docker compose exec [CONTAINER] bash @@ -92,7 +90,7 @@ docker compose exec [CONTAINER] bash ### Setup Live Editing for the Web UI Use a `docker-compose.override.yml` to mount the local `python` directory to -`/home/pi/RASCSI/python/` in the `rascsi_web` container. +`/home/pi/RASCSI/python/` in the `web` container. Any changes to *.py files on the host computer (i.e. in your IDE) will trigger the web UI process to be restarted in the container. @@ -100,7 +98,7 @@ the web UI process to be restarted in the container. **Example:** ``` services: - rascsi_web: + web: volumes: - ../python:/home/pi/RASCSI/python:delegated ``` diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile new file mode 100644 index 00000000..7245484e --- /dev/null +++ b/docker/backend/Dockerfile @@ -0,0 +1,36 @@ +ARG DEBIAN_FRONTEND=noninteractive + +FROM debian:bullseye AS build +RUN apt-get update && apt-get install --assume-yes --no-install-recommends sudo +RUN groupadd pi \ + && useradd --create-home --shell /bin/bash -g pi pi \ + && echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +USER pi +WORKDIR /home/pi/RASCSI + +COPY --chown=pi:pi easyinstall.sh . +COPY --chown=pi:pi cpp cpp +COPY --chown=pi:pi doc doc +RUN ./easyinstall.sh --run_choice=15 --cores=`nproc` + +FROM debian:bullseye-slim AS runner +USER root +WORKDIR /home/pi + +COPY --from=build /home/pi/RASCSI/cpp/bin/fullspec/* /usr/local/bin/ +COPY docker/backend/rascsi_wrapper.sh /usr/local/bin/rascsi_wrapper.sh +RUN chmod +x /usr/local/bin/* +RUN mkdir -p /home/pi/images + +RUN apt-get update \ + && apt-get install --no-install-recommends --assume-yes libpcap-dev libprotobuf-dev \ + && apt autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 6868 +ENTRYPOINT ["/usr/local/bin/rascsi_wrapper.sh", "-r", "7", "-F", "/home/pi/images"] +CMD ["-L", "trace"] + +HEALTHCHECK --interval=5m --timeout=1s CMD rasctl -v diff --git a/docker/rascsi/rascsi_wrapper.sh b/docker/backend/rascsi_wrapper.sh similarity index 100% rename from docker/rascsi/rascsi_wrapper.sh rename to docker/backend/rascsi_wrapper.sh diff --git a/docker/docker-compose.ci.yml b/docker/docker-compose.ci.yml new file mode 100644 index 00000000..db5866f3 --- /dev/null +++ b/docker/docker-compose.ci.yml @@ -0,0 +1,42 @@ +services: + backend: + image: ${DOCKER_BACKEND_IMAGE} + build: + context: .. + dockerfile: docker/backend/Dockerfile + init: true + volumes: + - ./volumes/images:/home/pi/images:delegated + healthcheck: + interval: 5s + start_period: 5s + + web: + build: + context: .. + dockerfile: docker/web/Dockerfile + args: + - OS_VERSION=buster + volumes: + - ./volumes/images:/home/pi/images:delegated + init: true + command: ["--rascsi-host=backend", "--log-level=debug"] + healthcheck: + interval: 5s + start_period: 5s + + pytest: + depends_on: + web: + condition: service_healthy + backend: + condition: service_healthy + profiles: + - webui-tests + build: + context: .. + dockerfile: docker/pytest/Dockerfile + working_dir: /src + volumes: + - ./volumes/pytest:/src/tests/output:delegated + command: ["-vv"] \ No newline at end of file diff --git a/docker/docker-compose.override.yml.example b/docker/docker-compose.override.yml.example index 725fdda5..53134ebb 100644 --- a/docker/docker-compose.override.yml.example +++ b/docker/docker-compose.override.yml.example @@ -1,4 +1,7 @@ services: - rascsi_web: + web: volumes: - ../python:/home/pi/RASCSI/python:delegated + pytest: + volumes: + - ../python/web:/src:delegated diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index df572f19..dc93b3dd 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,15 +1,11 @@ services: - rascsi: - container_name: rascsi - image: rascsi:develop-${OS_DISTRO:-debian}-${OS_VERSION:-buster}-${OS_ARCH:-amd64}-standalone + backend: + container_name: rascsi_backend + image: rascsi-backend pull_policy: never build: context: .. - dockerfile: docker/rascsi/Dockerfile - args: - - OS_DISTRO=${OS_DISTRO:-debian} - - OS_VERSION=${OS_VERSION:-buster} - - OS_ARCH=${OS_ARCH:-amd64} + dockerfile: docker/backend/Dockerfile volumes: - ./volumes/images:/home/pi/images:delegated - ./volumes/config:/home/pi/.config/rascsi:delegated @@ -19,26 +15,19 @@ services: - RASCSI_PASSWORD=${RASCSI_PASSWORD:-} init: true command: [ - "/usr/local/bin/rascsi_wrapper.sh", "-L", "${RASCSI_LOG_LEVEL:-trace}", - "-r", - "7", - "-F", - "/home/pi/images" ] - rascsi_web: + web: container_name: rascsi_web - image: rascsi:develop-${OS_DISTRO:-debian}-${OS_VERSION:-buster}-${OS_ARCH:-amd64}-web + image: rascsi-web:${OS_VERSION:-buster} pull_policy: never build: context: .. - dockerfile: docker/rascsi-web/Dockerfile + dockerfile: docker/web/Dockerfile args: - - OS_DISTRO=${OS_DISTRO:-debian} - OS_VERSION=${OS_VERSION:-buster} - - OS_ARCH=${OS_ARCH:-amd64} volumes: - ./volumes/images:/home/pi/images:delegated - ./volumes/config:/home/pi/.config/rascsi:delegated @@ -49,23 +38,20 @@ services: - RASCSI_PASSWORD=${RASCSI_PASSWORD:-} init: true command: [ - "start.sh", - "--rascsi-host=${RASCSI_HOST:-rascsi}", + "--rascsi-host=${RASCSI_HOST:-backend}", "--rascsi-port=${RASCSI_PORT:-6868}", - "--log-level=${WEB_LOG_LEVEL:-info}", + "--log-level=${WEB_LOG_LEVEL:-debug}", "--dev-mode" ] pytest: - container_name: pytest - image: rascsi:pytest + container_name: rascsi_pytest + image: rascsi-pytest pull_policy: never profiles: - webui-tests build: context: .. dockerfile: docker/pytest/Dockerfile - volumes: - - ../python/web:/src:delegated working_dir: /src - entrypoint: "pytest" + command: ["-vv"] \ No newline at end of file diff --git a/docker/pytest/Dockerfile b/docker/pytest/Dockerfile index 712b1cda..10bda346 100644 --- a/docker/pytest/Dockerfile +++ b/docker/pytest/Dockerfile @@ -1,5 +1,12 @@ -FROM python:3.7-bullseye +FROM python:3.7-slim ENV DOCKER=1 -COPY python/web/requirements-dev.txt /requirements-dev.txt -RUN pip install -r /requirements-dev.txt +WORKDIR /src + +COPY python/web/requirements-dev.txt /src/requirements-dev.txt +COPY python/web/pyproject.toml /src/pyproject.toml +COPY python/web/tests /src/tests + +RUN pip install --no-cache-dir -r /src/requirements-dev.txt + +ENTRYPOINT ["pytest"] diff --git a/docker/rascsi-web/Dockerfile b/docker/rascsi-web/Dockerfile deleted file mode 100644 index f3f4c367..00000000 --- a/docker/rascsi-web/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -ARG OS_DISTRO=debian -ARG OS_VERSION=buster -ARG OS_ARCH=amd64 -FROM "${OS_ARCH}/${OS_DISTRO}:${OS_VERSION}" - -EXPOSE 80 443 - -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - sudo \ - systemd \ - rsyslog \ - procps \ - man-db \ - wget \ - git - -RUN groupadd pi -RUN useradd --create-home --shell /bin/bash -g pi pi -RUN echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers -RUN echo "pi:rascsi" | chpasswd - -# Allows custom PATH for mock commands to work when executing with sudo -RUN sed -i 's/^Defaults\tsecure_path/#Defaults\tsecure_path./' /etc/sudoers - -RUN mkdir /home/pi/shared_files -RUN touch /etc/dhcpcd.conf -RUN mkdir -p /etc/network/interfaces.d/ - -USER pi -WORKDIR /home/pi/RASCSI -COPY --chown=pi:pi . . - -# Install standalone RaSCSI web UI -RUN ./easyinstall.sh --run_choice=11 - -# Enable web UI authentication -RUN ./easyinstall.sh --run_choice=13 - -# Setup wired network bridge -RUN ./easyinstall.sh --run_choice=5 --headless - -USER root -WORKDIR /home/pi -RUN pip3 install watchdog -COPY docker/rascsi-web/start.sh /usr/local/bin/start.sh -RUN chmod +x /usr/local/bin/start.sh -CMD ["/usr/local/bin/start.sh"] diff --git a/docker/rascsi/Dockerfile b/docker/rascsi/Dockerfile deleted file mode 100644 index 174846ad..00000000 --- a/docker/rascsi/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -ARG OS_DISTRO=debian -ARG OS_VERSION=buster -ARG OS_ARCH=amd64 -FROM "${OS_ARCH}/${OS_DISTRO}:${OS_VERSION}" - -EXPOSE 6868 - -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends sudo systemd rsyslog patch wget - -RUN groupadd pi -RUN useradd --create-home --shell /bin/bash -g pi pi -RUN echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers - -USER pi -WORKDIR /home/pi/RASCSI -COPY --chown=pi:pi . . - -# Install RaSCSI standalone -RUN ./easyinstall.sh --run_choice=10 --cores=`nproc` - -USER root -WORKDIR /home/pi -COPY docker/rascsi/rascsi_wrapper.sh /usr/local/bin/rascsi_wrapper.sh -RUN chmod +x /usr/local/bin/rascsi_wrapper.sh -CMD ["/usr/local/bin/rascsi_wrapper.sh", "-L", "trace", "-r", "7", "-F", "/home/pi/images"] diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile new file mode 100644 index 00000000..48e958f6 --- /dev/null +++ b/docker/web/Dockerfile @@ -0,0 +1,57 @@ +ARG DEBIAN_FRONTEND=noninteractive +ARG OS_VERSION=buster + +FROM "debian:${OS_VERSION}-slim" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends sudo systemd rsyslog procps man-db wget git \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd pi \ + && useradd --create-home --shell /bin/bash -g pi pi \ + && echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \ + && echo "pi:rascsi" | chpasswd + +# Allows custom PATH for mock commands to work when executing with sudo +RUN sed -i 's/^Defaults\tsecure_path/#Defaults\tsecure_path./' /etc/sudoers + +RUN mkdir -p /home/pi/shared_files \ + && mkdir /home/pi/images \ + && mkdir -p /etc/network/interfaces.d \ + && touch /etc/dhcpcd.conf + +USER pi +WORKDIR /home/pi/RASCSI + +RUN mkdir /home/pi/RASCSI/{python,cpp} +COPY --chown=pi:pi easyinstall.sh . +COPY --chown=pi:pi cpp/os_integration cpp/os_integration +COPY --chown=pi:pi cpp/rascsi_interface.proto cpp/rascsi_interface.proto +COPY --chown=pi:pi python/web python/web +COPY --chown=pi:pi python/common python/common + +# Install standalone RaSCSI web UI +RUN ./easyinstall.sh --run_choice=11 \ + && sudo apt-get remove build-essential --yes \ + && sudo apt autoremove -y \ + && sudo apt-get clean \ + && sudo rm -rf /var/lib/apt/lists/* + +# Enable web UI authentication +RUN ./easyinstall.sh --run_choice=13 + +# Setup wired network bridge +RUN ./easyinstall.sh --run_choice=5 --headless + +USER root +WORKDIR /home/pi +RUN pip3 install --no-cache-dir PyYAML watchdog +COPY docker/web/start.sh /usr/local/bin/start.sh +RUN chmod +x /usr/local/bin/start.sh + +EXPOSE 80 443 +ENTRYPOINT ["/usr/local/bin/start.sh"] + +HEALTHCHECK --interval=5m --timeout=3s \ + CMD wget --quiet --server-response http://localhost/healthcheck 2>&1 | grep "200 OK" diff --git a/docker/rascsi-web/start.sh b/docker/web/start.sh similarity index 92% rename from docker/rascsi-web/start.sh rename to docker/web/start.sh index 19bc0399..3b027018 100644 --- a/docker/rascsi-web/start.sh +++ b/docker/web/start.sh @@ -3,7 +3,7 @@ if ! [[ -f "/home/pi/RASCSI/python/common/src/rascsi_interface_pb2.py" ]]; then # Build rascsi_interface_pb2.py with the protobuf compiler protoc \ - -I=/home/pi/RASCSI/src/raspberrypi \ + -I=/home/pi/RASCSI/cpp \ --python_out=/home/pi/RASCSI/python/common/src \ rascsi_interface.proto fi diff --git a/easyinstall.sh b/easyinstall.sh index 2b960a7f..0458b8fe 100755 --- a/easyinstall.sh +++ b/easyinstall.sh @@ -69,6 +69,11 @@ SECRET_FILE="$HOME/.config/rascsi/rascsi_secret" FILE_SHARE_PATH="$HOME/shared_files" FILE_SHARE_NAME="Pi File Server" +APT_PACKAGES_COMMON="build-essential git protobuf-compiler bridge-utils" +APT_PACKAGES_BACKEND="libspdlog-dev libpcap-dev libprotobuf-dev protobuf-compiler libgmock-dev clang-11" +APT_PACKAGES_PYTHON="python3 python3-dev python3-pip python3-venv python3-setuptools python3-wheel libev-dev libevdev2" +APT_PACKAGES_WEB="nginx-light genisoimage man2html hfsutils dosfstools kpartx unzip unar disktype" + set -e # checks to run before entering the script main menu @@ -96,51 +101,36 @@ function installPackages() { return 0 fi sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y -qq \ - build-essential \ - git \ - libspdlog-dev \ - libpcap-dev \ - libprotobuf-dev \ - genisoimage \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv \ - python3-setuptools \ - python3-wheel \ - nginx-light \ - protobuf-compiler \ - bridge-utils \ - libev-dev \ - libevdev2 \ - unzip \ - unar \ - disktype \ - libgmock-dev \ - man2html \ - hfsutils \ - dosfstools \ - kpartx \ - clang-11 + $APT_PACKAGES_COMMON \ + $APT_PACKAGES_BACKEND \ + $APT_PACKAGES_PYTHON \ + $APT_PACKAGES_WEB } -# install Debian packges for RaSCSI standalone +# install Debian packages for RaSCSI standalone function installPackagesStandalone() { if [[ $SKIP_PACKAGES ]]; then echo "Skipping package installation" return 0 fi sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y -qq \ - build-essential \ - git \ - libspdlog-dev \ - libpcap-dev \ - libprotobuf-dev \ - protobuf-compiler \ - libgmock-dev \ - clang-11 + $APT_PACKAGES_COMMON \ + $APT_PACKAGES_BACKEND } +# install Debian packages for RaSCSI web UI standalone +function installPackagesWeb() { + if [[ $SKIP_PACKAGES ]]; then + echo "Skipping package installation" + return 0 + fi + sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y -qq \ + $APT_PACKAGES_COMMON \ + $APT_PACKAGES_PYTHON \ + $APT_PACKAGES_WEB +} + + # cache the pip packages function cachePipPackages(){ pushd $WEB_INSTALL_PATH @@ -1363,7 +1353,7 @@ function runChoice() { sudoCheck createCfgDir updateRaScsiGit - installPackages + installPackagesWeb installHfdisk fetchHardDiskDrivers preparePythonCommon @@ -1394,6 +1384,10 @@ function runChoice() { shareImagesWithNetatalk echo "Configuring AppleShare File Server - Complete!" ;; + 15) + installPackagesStandalone + compileRaScsi + ;; -h|--help|h|help) showMenu ;; @@ -1439,6 +1433,7 @@ function showMenu() { echo " 13) Enable or disable RaSCSI Web Interface authentication" echo "EXPERIMENTAL FEATURES" echo " 14) Share the images dir over AppleShare (requires Netatalk)" + echo " 15) Compile RaSCSI binaries" } # parse arguments passed to the script @@ -1454,7 +1449,7 @@ while [ "$1" != "" ]; do CONNECT_TYPE=$VALUE ;; -r | --run_choice) - if ! [[ $VALUE =~ ^[1-9][0-9]?$ && $VALUE -ge 1 && $VALUE -le 14 ]]; then + if ! [[ $VALUE =~ ^[1-9][0-9]?$ && $VALUE -ge 1 && $VALUE -le 15 ]]; then echo "ERROR: The run choice parameter must have a numeric value between 1 and 14" exit 1 fi diff --git a/python/common/src/rascsi/file_cmds.py b/python/common/src/rascsi/file_cmds.py index ff9ca0be..f456ebd6 100644 --- a/python/common/src/rascsi/file_cmds.py +++ b/python/common/src/rascsi/file_cmds.py @@ -47,6 +47,13 @@ class FileCmds: self.token = token self.locale = locale + def send_pb_command(self, command): + if logging.getLogger().isEnabledFor(logging.DEBUG): + # TODO: Uncouple/move to common dependency + logging.debug(self.ractl.format_pb_command(command)) + + return self.sock_cmd.send_pb_command(command.SerializeToString()) + # noinspection PyMethodMayBeStatic # pylint: disable=no-self-use def list_files(self, file_types, dir_path): @@ -89,7 +96,7 @@ class FileCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) @@ -168,7 +175,7 @@ class FileCmds: command.params["size"] = str(size) command.params["read_only"] = "false" - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} @@ -186,7 +193,7 @@ class FileCmds: command.params["file"] = file_name - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} @@ -205,7 +212,7 @@ class FileCmds: command.params["from"] = file_name command.params["to"] = new_file_name - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} @@ -224,7 +231,7 @@ class FileCmds: command.params["from"] = file_name command.params["to"] = new_file_name - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} diff --git a/python/common/src/rascsi/ractl_cmds.py b/python/common/src/rascsi/ractl_cmds.py index a58a5f00..f36b22a7 100644 --- a/python/common/src/rascsi/ractl_cmds.py +++ b/python/common/src/rascsi/ractl_cmds.py @@ -5,6 +5,7 @@ Module for commands sent to the RaSCSI backend service. import rascsi_interface_pb2 as proto from rascsi.return_codes import ReturnCodes from rascsi.socket_cmds import SocketCmds +import logging class RaCtlCmds: @@ -17,6 +18,12 @@ class RaCtlCmds: self.token = token self.locale = locale + def send_pb_command(self, command): + if logging.getLogger().isEnabledFor(logging.DEBUG): + logging.debug(self.format_pb_command(command)) + + return self.sock_cmd.send_pb_command(command.SerializeToString()) + def get_server_info(self): """ Sends a SERVER_INFO command to the server. @@ -35,7 +42,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) version = ( @@ -93,7 +100,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) scsi_ids = [] @@ -114,7 +121,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) ifs = result.network_interfaces_info.name @@ -133,7 +140,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) device_types = {} @@ -199,7 +206,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.token - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) images_dir = result.image_files_info.default_image_folder @@ -273,7 +280,7 @@ class RaCtlCmds: command.devices.append(devices) - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} @@ -295,7 +302,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} @@ -310,7 +317,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} @@ -332,7 +339,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} @@ -360,7 +367,7 @@ class RaCtlCmds: device.unit = int(unit) command.devices.append(device) - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) @@ -430,7 +437,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} @@ -447,7 +454,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} @@ -466,7 +473,7 @@ class RaCtlCmds: command.params["token"] = self.token command.params["locale"] = self.locale - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} @@ -480,7 +487,37 @@ class RaCtlCmds: command = proto.PbCommand() command.operation = proto.PbOperation.CHECK_AUTHENTICATION - data = self.sock_cmd.send_pb_command(command.SerializeToString()) + data = self.send_pb_command(command) result = proto.PbResult() result.ParseFromString(data) return {"status": result.status, "msg": result.msg} + + def format_pb_command(self, command): + """ + Formats the Protobuf command for output + """ + message = f"Sending: {proto.PbOperation.Name(command.operation)}" + + params = { + name: "***" if name == "token" else value + for (name, value) in sorted(command.params.items()) + } + message += f", params: {params}" + + for device in command.devices: + formatted_device = { + key: value + for (key, value) in { + "id": device.id, + "unit": device.unit, + "type": proto.PbDeviceType.Name(device.type) if device.type else None, + "params": device.params, + "vendor": device.vendor, + "product": device.product, + "revision": device.revision, + }.items() + if key == "id" or value + } + message += f", device: {formatted_device}" + + return message diff --git a/python/web/mock/bin/brctl b/python/web/mock/bin/brctl old mode 100644 new mode 100755 diff --git a/python/web/mock/bin/git b/python/web/mock/bin/git new file mode 100755 index 00000000..742e13d6 --- /dev/null +++ b/python/web/mock/bin/git @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exit 0 diff --git a/python/web/mock/bin/journalctl b/python/web/mock/bin/journalctl old mode 100644 new mode 100755 diff --git a/python/web/mock/bin/systemctl b/python/web/mock/bin/systemctl old mode 100644 new mode 100755 diff --git a/python/web/pyproject.toml b/python/web/pyproject.toml index 25fc8772..4ea67511 100644 --- a/python/web/pyproject.toml +++ b/python/web/pyproject.toml @@ -1,4 +1,4 @@ [tool.pytest.ini_options] -addopts = "--junitxml=report.xml" +addopts = "--junitxml=tests/output/report.xml --log-file=tests/output/pytest.log" log_cli = true log_cli_level = "warn" \ No newline at end of file diff --git a/python/web/src/templates/index.html b/python/web/src/templates/index.html index 3653af3a..f8675d18 100644 --- a/python/web/src/templates/index.html +++ b/python/web/src/templates/index.html @@ -709,7 +709,11 @@