Add GitHub Actions for web UI, fix C++ unit test job, enable SonarCloud cache, introduce parallel execution where possible (#974)

2024-12-21 08:29:59 +00:00 · 2022-11-18 13:58:26 +00:00 · 2022-11-18 13:58:26 +00:00 · 163cb5a10e
commit 163cb5a10e
parent 4afb11d3dd
5 changed files with 246 additions and 110 deletions
--- a/.github/workflows/arm_cross_compile.yml
+++ b/.github/workflows/arm_cross_compile.yml
@ -0,0 +1,52 @@
+on:
+  workflow_call:
+    inputs:
+      connect-type:
+        required: true
+        type: string
+
+jobs:
+  build_arm:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: cpp
+    env:
+      APT_ARM_TOOLCHAIN: "gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf binutils-arm-linux-gnueabihf libspdlog-dev"
+      APT_LIBRARIES: "libspdlog-dev:armhf libpcap-dev:armhf libevdev2:armhf libev-dev:armhf protobuf-compiler libprotobuf-dev:armhf"
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Add armhf as architecture
+        run: sudo dpkg --add-architecture armhf
+
+      - name: Reconfigure apt for arch amd64
+        run: sudo sed -i "s/deb /deb [arch=amd64] /g" /etc/apt/sources.list
+
+      - name: Add armhf repos (jammy)
+        run: sudo bash -c "echo \"deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ jammy main multiverse restricted universe\" >> /etc/apt/sources.list"
+
+      - name: Add armhf repos (jammy-updates)
+        run: sudo bash -c "echo \"deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main multiverse restricted universe\" >> /etc/apt/sources.list"
+
+      - name: Update apt
+        run: sudo apt update
+
+      - name: Install apt packages
+        run: sudo apt-get --yes install ${{ env.APT_ARM_TOOLCHAIN }} ${{ env.APT_LIBRARIES }}
+
+      - name: Compile
+        run: make all -j 6 CONNECT_TYPE=${{ inputs.connect-type }} CROSS_COMPILE=arm-linux-gnueabihf-
+
+      # We need to tar the binary outputs to retain the executable
+      # file permission. Currently, actions/upload-artifact only
+      # supports .ZIP files.
+      # This is workaround for https://github.com/actions/upload-artifact/issues/38
+      - name: Tar binary output
+        run: tar -czvf rascsi-${{ inputs.connect-type }}.tar.gz ./bin
+
+      - name: Upload binaries
+        uses: actions/upload-artifact@v3
+        with:
+          name: arm-binaries-${{ inputs.connect-type }}.tar.gz
+          path: cpp/rascsi-${{ inputs.connect-type }}.tar.gz
--- a/.github/workflows/build_code.yml
+++ b/.github/workflows/build_code.yml
@ -1,54 +1,20 @@
-name: Build C++ Packages
+name: Build ARM binaries

-on: [push, pull_request]
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - 'cpp/**'
+      - '.github/workflows/build_code.yml'

 jobs:
-  build:
-    runs-on: ubuntu-22.04
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Add armhf as architecture
-      run: sudo dpkg --add-architecture armhf
-
-    - name: Reconfigure apt for arch amd64
-      run: sudo sed -i "s/deb /deb [arch=amd64] /g" /etc/apt/sources.list
-
-    - name: Add armhf repos (jammy)
-      run: sudo bash -c "echo \"deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ jammy main multiverse restricted universe\" >> /etc/apt/sources.list"
-
-    - name: Add armhf repos (jammy-updates)
-      run: sudo bash -c "echo \"deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main multiverse restricted universe\" >> /etc/apt/sources.list"
-
-    - name: Update apt
-      run: sudo apt update
-
-    - name: Install cross compile toolchain
-      run: sudo apt-get --yes install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf binutils-arm-linux-gnueabihf libspdlog-dev
-    
-    - name: Install libraries
-      run: sudo apt-get --yes install libspdlog-dev:armhf libpcap-dev:armhf libevdev2:armhf libev-dev:armhf protobuf-compiler libprotobuf-dev:armhf 
-
-    - name: make standard
-      run: make all -j6 CONNECT_TYPE=STANDARD CROSS_COMPILE=arm-linux-gnueabihf-
-      working-directory: ./cpp
- 
-    - name: make fullspec
-      run: make all -j6 CONNECT_TYPE=FULLSPEC CROSS_COMPILE=arm-linux-gnueabihf-
-      working-directory: ./cpp
-
-    # We need to tar the binary outputs to retain the executable
-    # file permission. Currently, actions/upload-artifact only
-    # supports .ZIP files.
-    # This is workaround for https://github.com/actions/upload-artifact/issues/38
-    - name: tar binary outputs
-      run: tar -czvf rascsi.tar.gz ./bin
-      working-directory: ./cpp
-
-    - name: upload artifacts
-      uses: actions/upload-artifact@v2
+  fullspec:
+    uses: akuker/RASCSI/.github/workflows/arm_cross_compile.yml@develop
    with:
-        name: arm-binaries
-        path: ./cpp/rascsi.tar.gz
+      connect-type: "FULLSPEC"

+  standard:
+    uses: akuker/RASCSI/.github/workflows/arm_cross_compile.yml@develop
+    with:
+      connect-type: "STANDARD"
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@ -0,0 +1,122 @@
+name: C++ Tests/Analysis
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - 'cpp/**'
+      - '.github/workflows/cpp.yml'
+
+env:
+  APT_PACKAGES: libspdlog-dev libpcap-dev libevdev2 libev-dev protobuf-compiler libgtest-dev libgmock-dev
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: cpp
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: sudo apt-get install ${{ env.APT_PACKAGES }}
+
+      - name: Build unit tests
+        run: make -j $(nproc) test
+
+      - name: Run unit tests
+        run: (set -o pipefail && bin/fullspec/rascsi_test | tee rascsi_test_log.txt)
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        with:
+          name: rascsi_test_log.txt
+          path: cpp/rascsi_test_log.txt
+
+  sonarcloud:
+    runs-on: ubuntu-22.04
+    if: github.repository == 'akuker/RASCSI'
+    env:
+      SOURCES: cpp
+      BUILD_WRAPPER_OUT_DIR: "$HOME/.build_wrapper_out" # Directory where build-wrapper output will be placed
+      SONAR_SCANNER_VERSION: 4.7.0.2747
+      SONAR_SERVER_URL: "https://sonarcloud.io"
+      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+      SONAR_PROJECT_KEY: "akuker_RASCSI"
+      SONAR_ORGANIZATION: "rascsi"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
+
+      - name: Install dependencies
+        run: sudo apt-get install ${{ env.APT_PACKAGES }}
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v1
+        with:
+          java-version: 17
+
+      - uses: actions/cache@v3
+        name: Cache SonarCloud setup
+        id: sonar-install-cache
+        with:
+          path: ~/.sonar
+          key: sonar-with-build-wrapper-${{ env.SONAR_SCANNER_VERSION }}
+
+      - name: Set up SonarCloud scanner
+        if: steps.sonar-install-cache.outputs.cache-hit != 'true'
+        env:
+          SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip
+        run: |
+          mkdir -p $HOME/.sonar
+          curl -sSLo /tmp/sonar-scanner.zip ${{ env.SONAR_SCANNER_DOWNLOAD_URL }}
+          unzip -o /tmp/sonar-scanner.zip -d $HOME/.sonar/
+
+      - name: Set up SonarCloud build wrapper
+        if: steps.sonar-install-cache.outputs.cache-hit != 'true'
+        env:
+          BUILD_WRAPPER_DOWNLOAD_URL: ${{ env.SONAR_SERVER_URL }}/static/cpp/build-wrapper-linux-x86.zip
+        run: |
+          curl -sSLo /tmp/build-wrapper-linux-x86.zip ${{ env.BUILD_WRAPPER_DOWNLOAD_URL }}
+          unzip -o /tmp/build-wrapper-linux-x86.zip -d $HOME/.sonar/
+
+      - name: Generate coverage
+        run: >-
+          (mkdir -p ${{ env.BUILD_WRAPPER_OUT_DIR }} || true) &&
+          $HOME/.sonar/build-wrapper-linux-x86/build-wrapper-linux-x86-64
+          --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }}
+          make -j $(nproc) -C $SOURCES coverage
+
+      - name: Run gcov
+        working-directory: cpp
+        run: gcov --preserve-paths $(find -name '*.gcno')
+
+      - uses: actions/cache@v3
+        name: Cache SonarCloud scan cache
+        id: sonar-scan-cache
+        with:
+          path: ~/.sonar_cache/
+          key: sonar-scan-cache-${{ env.SONAR_SCANNER_VERSION }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
+          restore-keys: |
+            sonar-scan-cache-${{ env.SONAR_SCANNER_VERSION }}-${{ github.ref_name }}
+
+      - name: Run sonar-scanner
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: >-
+          (mkdir -p $HOME/.sonar_cache || true) &&
+          $HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin/sonar-scanner
+          --define sonar.host.url="${{ env.SONAR_SERVER_URL }}"
+          --define sonar.projectKey=${{ env.SONAR_PROJECT_KEY }}
+          --define sonar.organization=${{ env.SONAR_ORGANIZATION }}
+          --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}"
+          --define sonar.cfamily.gcov.reportsPath=.
+          --define sonar.cfamily.cache.enabled=true
+          --define sonar.cfamily.cache.path="$HOME/.sonar_cache/"
+          --define sonar.coverage.exclusions="cpp/**/test/**"
+          --define sonar.cpd.exclusions="cpp/**/test/**"
+          --define sonar.inclusions="cpp/**,python/**"
+          --define sonar.python.version=3.7,3.9
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@ -1,60 +0,0 @@
-name: Run automated unit tests
-
-on: [push, pull_request]
-
-jobs:
-  Tests_and_SonarCloud_Analysis:
-    runs-on: ubuntu-22.04
-    env:
-      MAKEFLAGS: -j2 # Number of available processors
-      SOURCES: cpp
-      SONAR_SCANNER_VERSION: 4.7.0.2747
-      SONAR_SERVER_URL: "https://sonarcloud.io"
-      BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
-      - name: Install dependencies
-        run: sudo apt-get install libspdlog-dev libpcap-dev libevdev2 libev-dev protobuf-compiler libgtest-dev libgmock-dev
-
-      - name: Run unit tests and save log
-        run: $SOURCES/bin/fullspec/rascsi_test | tee test_log.txt
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: test log
-          path: test_log.txt
-
-      - name: Set up JDK 17
-        uses: actions/setup-java@v1
-        with:
-          java-version: 17
-      - name: Download and set up sonar-scanner
-        env:
-          SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip
-        run: |
-          mkdir -p $HOME/.sonar
-          curl -sSLo $HOME/.sonar/sonar-scanner.zip ${{ env.SONAR_SCANNER_DOWNLOAD_URL }}
-          unzip -o $HOME/.sonar/sonar-scanner.zip -d $HOME/.sonar/
-          echo "$HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin" >> $GITHUB_PATH
-      - name: Download and set up build-wrapper
-        env:
-          BUILD_WRAPPER_DOWNLOAD_URL: ${{ env.SONAR_SERVER_URL }}/static/cpp/build-wrapper-linux-x86.zip
-        run: |
-          curl -sSLo $HOME/.sonar/build-wrapper-linux-x86.zip ${{ env.BUILD_WRAPPER_DOWNLOAD_URL }}
-          unzip -o $HOME/.sonar/build-wrapper-linux-x86.zip -d $HOME/.sonar/
-          echo "$HOME/.sonar/build-wrapper-linux-x86" >> $GITHUB_PATH
-      - name: Run build-wrapper
-        run: |
-          build-wrapper-linux-x86-64 --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }} make -C $SOURCES coverage
-
-      - name: Run gcov
-        run: (cd $SOURCES ; gcov --preserve-paths $(find -name '*.gcno'))
-      - name: Run sonar-scanner
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        run: |
-          cd $SOURCES | sonar-scanner --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" --define sonar.projectKey=akuker_RASCSI --define sonar.organization=rascsi --define sonar.cfamily.gcov.reportsPath=. --define sonar.coverage.exclusions="**/test/**" --define sonar.cpd.exclusions="**test/**" --define sonar.python.version=3
-
--- a/.github/workflows/web.yml
+++ b/.github/workflows/web.yml
@ -0,0 +1,56 @@
+name: Web Tests/Analysis
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - 'python/web/**'
+      - 'python/common/**'
+      - '.github/workflows/web.yml'
+
+jobs:
+  backend_checks:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: python/web
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.7.15
+          cache: 'pip'
+
+      - run: pip install -r requirements-dev.txt
+        id: pip
+
+      - run: black --check tests
+
+      - run: flake8 tests
+        if: success() || failure() && steps.pip.outcome == 'success'
+
+  frontend_checks:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: python/web
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+        cache: 'npm'
+        cache-dependency-path: python/web/package-lock.json
+
+    - run: npm ci
+      id: npm
+
+    - name: Stylelint
+      run: npx stylelint src/static/themes/modern/style.css
+
+    - name: Prettier
+      run: npx prettier --check src/static/themes/modern/style.css
+      if: success() || failure() && steps.npm.outcome == 'success'