From 163cb5a10e3f41a94ead3abfe1965f330b93e357 Mon Sep 17 00:00:00 2001 From: nucleogenic Date: Fri, 18 Nov 2022 13:58:26 +0000 Subject: [PATCH] Add GitHub Actions for web UI, fix C++ unit test job, enable SonarCloud cache, introduce parallel execution where possible (#974) --- .github/workflows/arm_cross_compile.yml | 52 ++++++++++ .github/workflows/build_code.yml | 66 ++++--------- .github/workflows/cpp.yml | 122 ++++++++++++++++++++++++ .github/workflows/run_tests.yml | 60 ------------ .github/workflows/web.yml | 56 +++++++++++ 5 files changed, 246 insertions(+), 110 deletions(-) create mode 100644 .github/workflows/arm_cross_compile.yml create mode 100644 .github/workflows/cpp.yml delete mode 100644 .github/workflows/run_tests.yml create mode 100644 .github/workflows/web.yml diff --git a/.github/workflows/arm_cross_compile.yml b/.github/workflows/arm_cross_compile.yml new file mode 100644 index 00000000..92df712d --- /dev/null +++ b/.github/workflows/arm_cross_compile.yml @@ -0,0 +1,52 @@ +on: + workflow_call: + inputs: + connect-type: + required: true + type: string + +jobs: + build_arm: + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: cpp + env: + APT_ARM_TOOLCHAIN: "gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf binutils-arm-linux-gnueabihf libspdlog-dev" + APT_LIBRARIES: "libspdlog-dev:armhf libpcap-dev:armhf libevdev2:armhf libev-dev:armhf protobuf-compiler libprotobuf-dev:armhf" + steps: + - uses: actions/checkout@v3 + + - name: Add armhf as architecture + run: sudo dpkg --add-architecture armhf + + - name: Reconfigure apt for arch amd64 + run: sudo sed -i "s/deb /deb [arch=amd64] /g" /etc/apt/sources.list + + - name: Add armhf repos (jammy) + run: sudo bash -c "echo \"deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ jammy main multiverse restricted universe\" >> /etc/apt/sources.list" + + - name: Add armhf repos (jammy-updates) + run: sudo bash -c "echo \"deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main multiverse restricted universe\" >> /etc/apt/sources.list" + + - name: Update apt + run: sudo apt update + + - name: Install apt packages + run: sudo apt-get --yes install ${{ env.APT_ARM_TOOLCHAIN }} ${{ env.APT_LIBRARIES }} + + - name: Compile + run: make all -j 6 CONNECT_TYPE=${{ inputs.connect-type }} CROSS_COMPILE=arm-linux-gnueabihf- + + # We need to tar the binary outputs to retain the executable + # file permission. Currently, actions/upload-artifact only + # supports .ZIP files. + # This is workaround for https://github.com/actions/upload-artifact/issues/38 + - name: Tar binary output + run: tar -czvf rascsi-${{ inputs.connect-type }}.tar.gz ./bin + + - name: Upload binaries + uses: actions/upload-artifact@v3 + with: + name: arm-binaries-${{ inputs.connect-type }}.tar.gz + path: cpp/rascsi-${{ inputs.connect-type }}.tar.gz diff --git a/.github/workflows/build_code.yml b/.github/workflows/build_code.yml index 5ed9acd7..f4bba01d 100644 --- a/.github/workflows/build_code.yml +++ b/.github/workflows/build_code.yml @@ -1,54 +1,20 @@ -name: Build C++ Packages +name: Build ARM binaries -on: [push, pull_request] +on: + workflow_dispatch: + pull_request: + types: [opened, synchronize] + paths: + - 'cpp/**' + - '.github/workflows/build_code.yml' jobs: - build: - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@v2 - - - name: Add armhf as architecture - run: sudo dpkg --add-architecture armhf - - - name: Reconfigure apt for arch amd64 - run: sudo sed -i "s/deb /deb [arch=amd64] /g" /etc/apt/sources.list - - - name: Add armhf repos (jammy) - run: sudo bash -c "echo \"deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ jammy main multiverse restricted universe\" >> /etc/apt/sources.list" - - - name: Add armhf repos (jammy-updates) - run: sudo bash -c "echo \"deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main multiverse restricted universe\" >> /etc/apt/sources.list" - - - name: Update apt - run: sudo apt update - - - name: Install cross compile toolchain - run: sudo apt-get --yes install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf binutils-arm-linux-gnueabihf libspdlog-dev - - - name: Install libraries - run: sudo apt-get --yes install libspdlog-dev:armhf libpcap-dev:armhf libevdev2:armhf libev-dev:armhf protobuf-compiler libprotobuf-dev:armhf - - - name: make standard - run: make all -j6 CONNECT_TYPE=STANDARD CROSS_COMPILE=arm-linux-gnueabihf- - working-directory: ./cpp - - - name: make fullspec - run: make all -j6 CONNECT_TYPE=FULLSPEC CROSS_COMPILE=arm-linux-gnueabihf- - working-directory: ./cpp - - # We need to tar the binary outputs to retain the executable - # file permission. Currently, actions/upload-artifact only - # supports .ZIP files. - # This is workaround for https://github.com/actions/upload-artifact/issues/38 - - name: tar binary outputs - run: tar -czvf rascsi.tar.gz ./bin - working-directory: ./cpp - - - name: upload artifacts - uses: actions/upload-artifact@v2 - with: - name: arm-binaries - path: ./cpp/rascsi.tar.gz + fullspec: + uses: akuker/RASCSI/.github/workflows/arm_cross_compile.yml@develop + with: + connect-type: "FULLSPEC" + standard: + uses: akuker/RASCSI/.github/workflows/arm_cross_compile.yml@develop + with: + connect-type: "STANDARD" diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml new file mode 100644 index 00000000..f1fba3ee --- /dev/null +++ b/.github/workflows/cpp.yml @@ -0,0 +1,122 @@ +name: C++ Tests/Analysis + +on: + workflow_dispatch: + pull_request: + types: [opened, synchronize] + paths: + - 'cpp/**' + - '.github/workflows/cpp.yml' + +env: + APT_PACKAGES: libspdlog-dev libpcap-dev libevdev2 libev-dev protobuf-compiler libgtest-dev libgmock-dev + +jobs: + unit_tests: + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: cpp + steps: + - uses: actions/checkout@v3 + + - name: Install dependencies + run: sudo apt-get install ${{ env.APT_PACKAGES }} + + - name: Build unit tests + run: make -j $(nproc) test + + - name: Run unit tests + run: (set -o pipefail && bin/fullspec/rascsi_test | tee rascsi_test_log.txt) + + - name: Upload logs + uses: actions/upload-artifact@v3 + with: + name: rascsi_test_log.txt + path: cpp/rascsi_test_log.txt + + sonarcloud: + runs-on: ubuntu-22.04 + if: github.repository == 'akuker/RASCSI' + env: + SOURCES: cpp + BUILD_WRAPPER_OUT_DIR: "$HOME/.build_wrapper_out" # Directory where build-wrapper output will be placed + SONAR_SCANNER_VERSION: 4.7.0.2747 + SONAR_SERVER_URL: "https://sonarcloud.io" + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + SONAR_PROJECT_KEY: "akuker_RASCSI" + SONAR_ORGANIZATION: "rascsi" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + + - name: Install dependencies + run: sudo apt-get install ${{ env.APT_PACKAGES }} + + - name: Set up JDK 17 + uses: actions/setup-java@v1 + with: + java-version: 17 + + - uses: actions/cache@v3 + name: Cache SonarCloud setup + id: sonar-install-cache + with: + path: ~/.sonar + key: sonar-with-build-wrapper-${{ env.SONAR_SCANNER_VERSION }} + + - name: Set up SonarCloud scanner + if: steps.sonar-install-cache.outputs.cache-hit != 'true' + env: + SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip + run: | + mkdir -p $HOME/.sonar + curl -sSLo /tmp/sonar-scanner.zip ${{ env.SONAR_SCANNER_DOWNLOAD_URL }} + unzip -o /tmp/sonar-scanner.zip -d $HOME/.sonar/ + + - name: Set up SonarCloud build wrapper + if: steps.sonar-install-cache.outputs.cache-hit != 'true' + env: + BUILD_WRAPPER_DOWNLOAD_URL: ${{ env.SONAR_SERVER_URL }}/static/cpp/build-wrapper-linux-x86.zip + run: | + curl -sSLo /tmp/build-wrapper-linux-x86.zip ${{ env.BUILD_WRAPPER_DOWNLOAD_URL }} + unzip -o /tmp/build-wrapper-linux-x86.zip -d $HOME/.sonar/ + + - name: Generate coverage + run: >- + (mkdir -p ${{ env.BUILD_WRAPPER_OUT_DIR }} || true) && + $HOME/.sonar/build-wrapper-linux-x86/build-wrapper-linux-x86-64 + --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }} + make -j $(nproc) -C $SOURCES coverage + + - name: Run gcov + working-directory: cpp + run: gcov --preserve-paths $(find -name '*.gcno') + + - uses: actions/cache@v3 + name: Cache SonarCloud scan cache + id: sonar-scan-cache + with: + path: ~/.sonar_cache/ + key: sonar-scan-cache-${{ env.SONAR_SCANNER_VERSION }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} + restore-keys: | + sonar-scan-cache-${{ env.SONAR_SCANNER_VERSION }}-${{ github.ref_name }} + + - name: Run sonar-scanner + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + (mkdir -p $HOME/.sonar_cache || true) && + $HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin/sonar-scanner + --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" + --define sonar.projectKey=${{ env.SONAR_PROJECT_KEY }} + --define sonar.organization=${{ env.SONAR_ORGANIZATION }} + --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" + --define sonar.cfamily.gcov.reportsPath=. + --define sonar.cfamily.cache.enabled=true + --define sonar.cfamily.cache.path="$HOME/.sonar_cache/" + --define sonar.coverage.exclusions="cpp/**/test/**" + --define sonar.cpd.exclusions="cpp/**/test/**" + --define sonar.inclusions="cpp/**,python/**" + --define sonar.python.version=3.7,3.9 diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml deleted file mode 100644 index e4b6d28c..00000000 --- a/.github/workflows/run_tests.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Run automated unit tests - -on: [push, pull_request] - -jobs: - Tests_and_SonarCloud_Analysis: - runs-on: ubuntu-22.04 - env: - MAKEFLAGS: -j2 # Number of available processors - SOURCES: cpp - SONAR_SCANNER_VERSION: 4.7.0.2747 - SONAR_SERVER_URL: "https://sonarcloud.io" - BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - name: Install dependencies - run: sudo apt-get install libspdlog-dev libpcap-dev libevdev2 libev-dev protobuf-compiler libgtest-dev libgmock-dev - - - name: Run unit tests and save log - run: $SOURCES/bin/fullspec/rascsi_test | tee test_log.txt - - name: Upload artifacts - uses: actions/upload-artifact@v2 - with: - name: test log - path: test_log.txt - - - name: Set up JDK 17 - uses: actions/setup-java@v1 - with: - java-version: 17 - - name: Download and set up sonar-scanner - env: - SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip - run: | - mkdir -p $HOME/.sonar - curl -sSLo $HOME/.sonar/sonar-scanner.zip ${{ env.SONAR_SCANNER_DOWNLOAD_URL }} - unzip -o $HOME/.sonar/sonar-scanner.zip -d $HOME/.sonar/ - echo "$HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin" >> $GITHUB_PATH - - name: Download and set up build-wrapper - env: - BUILD_WRAPPER_DOWNLOAD_URL: ${{ env.SONAR_SERVER_URL }}/static/cpp/build-wrapper-linux-x86.zip - run: | - curl -sSLo $HOME/.sonar/build-wrapper-linux-x86.zip ${{ env.BUILD_WRAPPER_DOWNLOAD_URL }} - unzip -o $HOME/.sonar/build-wrapper-linux-x86.zip -d $HOME/.sonar/ - echo "$HOME/.sonar/build-wrapper-linux-x86" >> $GITHUB_PATH - - name: Run build-wrapper - run: | - build-wrapper-linux-x86-64 --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }} make -C $SOURCES coverage - - - name: Run gcov - run: (cd $SOURCES ; gcov --preserve-paths $(find -name '*.gcno')) - - name: Run sonar-scanner - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - cd $SOURCES | sonar-scanner --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" --define sonar.projectKey=akuker_RASCSI --define sonar.organization=rascsi --define sonar.cfamily.gcov.reportsPath=. --define sonar.coverage.exclusions="**/test/**" --define sonar.cpd.exclusions="**test/**" --define sonar.python.version=3 - diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml new file mode 100644 index 00000000..1eb8e8b4 --- /dev/null +++ b/.github/workflows/web.yml @@ -0,0 +1,56 @@ +name: Web Tests/Analysis + +on: + workflow_dispatch: + pull_request: + types: [opened, synchronize] + paths: + - 'python/web/**' + - 'python/common/**' + - '.github/workflows/web.yml' + +jobs: + backend_checks: + runs-on: ubuntu-latest + defaults: + run: + working-directory: python/web + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: 3.7.15 + cache: 'pip' + + - run: pip install -r requirements-dev.txt + id: pip + + - run: black --check tests + + - run: flake8 tests + if: success() || failure() && steps.pip.outcome == 'success' + + frontend_checks: + runs-on: ubuntu-latest + defaults: + run: + working-directory: python/web + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: 16 + cache: 'npm' + cache-dependency-path: python/web/package-lock.json + + - run: npm ci + id: npm + + - name: Stylelint + run: npx stylelint src/static/themes/modern/style.css + + - name: Prettier + run: npx prettier --check src/static/themes/modern/style.css + if: success() || failure() && steps.npm.outcome == 'success'