Build with analysis tools #10958
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build with analysis tools | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| run_all_benchmarks: | |
| description: Run all benchmarks | |
| type: boolean | |
| default: false | |
| dev_image_tag: | |
| description: Tag of the ArcticDB development image to use for benchmark and code coverage flows | |
| type: string | |
| default: latest | |
| storage: | |
| description: Storage to run benchmarks against | |
| type: choice | |
| options: | |
| - 'LMDB' | |
| - 'REAL' | |
| - 'ALL' | |
| default: 'LMDB' | |
| suite_overwrite: | |
| description: Specify regular expression for specific tests to be executed | |
| type: string | |
| default: '' | |
| run_sanitizers: | |
| description: Run C++ tests with sanitizers enabled | |
| type: boolean | |
| default: false | |
| # Run on push to master, just against LMDB backend. This is so that PR builds have recent master benchmark results | |
| # to compare against. | |
| push: | |
| branches: | |
| - master | |
| schedule: # Schedule the job to run at 12 a.m. daily. This run uses all storages. | |
| - cron: '0 0 * * *' | |
| pull_request: | |
| paths-ignore: | |
| - "**/*.md" | |
| jobs: | |
| get_commits_to_benchmark: | |
| name: Get tag commits | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3.3.0 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get tags | |
| id: get_tags | |
| run: | | |
| python3 build_tooling/get_commits_for_benchmark.py ${{ inputs.benchmark_all_tags == true && '--benchmark_all_tags' || ''}} | |
| outputs: | |
| matrix: ${{ steps.get_tags.outputs.commits }} | |
| benchmark_commits: | |
| needs: [get_commits_to_benchmark] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| commits: ${{ fromJson(needs.get_commits_to_benchmark.outputs.matrix)}} | |
| name: Benchmark commit ${{ matrix.commits }} | |
| uses: ./.github/workflows/benchmark_commits.yml | |
| secrets: inherit | |
| with: | |
| commit: ${{ matrix.commits }} | |
| benchmark_all_tags: ${{ inputs.benchmark_all_tags || false }} | |
| dev_image_tag: ${{ inputs.dev_image_tag || 'latest' }} | |
| storage: ${{ github.ref_name == 'master' && 'ALL' || inputs.storage || 'LMDB'}} | |
| suite_overwrite: ${{ inputs.suite_overwrite || ''}} | |
| publish_benchmark_results_to_gh_pages: | |
| name: Publish benchmark results to gh-pages | |
| if: ${{ github.event_name == 'schedule' }} # only publish for the scheduled master builds | |
| needs: [benchmark_commits] | |
| runs-on: ubuntu-22.04 | |
| container: | |
| image: ubuntu:22.04 # Native runner doesn't allow setting up the ca softlinks required below | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Select Python | |
| uses: actions/setup-python@v4.7.1 | |
| with: | |
| python-version: "3.10" | |
| - name: Prepare environment | |
| shell: bash -el {0} | |
| run: | | |
| apt update | |
| apt install -y git | |
| python -m pip install arcticdb[Testing] "protobuf<6" | |
| - name: Setup softlink for SSL | |
| shell: bash -el {0} | |
| run: | | |
| mkdir -p /etc/pki/tls | |
| ln -s /usr/lib/ssl/certs /etc/pki/tls/certs | |
| ln -s /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt | |
| - uses: actions/checkout@v3.3.0 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.ARCTICDB_TEST_PAT }} | |
| - name: Set persistent storage variables | |
| uses: ./.github/actions/set_persistent_storage_env_vars | |
| with: | |
| bucket: "arcticdb-ci-benchmark-results" | |
| aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}" | |
| aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}" | |
| - name: Publish results to Github Pages | |
| shell: bash -el {0} | |
| run: | | |
| git config --global --add safe.directory /__w/ArcticDB/ArcticDB | |
| git config --global user.name "${GITHUB_ACTOR}" | |
| git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" | |
| python build_tooling/transform_asv_results.py --mode extract | |
| python -m asv publish -v | |
| python -m asv gh-pages -v --rewrite | |
| run-asv-check-script: | |
| name: Executes asv tests checks | |
| timeout-minutes: 120 | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/man-group/arcticdb-dev:${{ inputs.dev_image_tag || 'latest' }} | |
| volumes: | |
| - /:/mnt | |
| env: | |
| # 0 - uses S3 Cache, 1 - uses GHA cache | |
| # this way the external PRs can use the GHA cache | |
| SCCACHE_GHA_VERSION: ${{secrets.AWS_S3_ACCESS_KEY == null}} | |
| SCCACHE_BUCKET: arcticdb-ci-sccache-bucket | |
| SCCACHE_ENDPOINT: http://s3.eu-west-1.amazonaws.com | |
| SCCACHE_REGION: eu-west-1 | |
| SCCACHE_S3_USE_SSL: false | |
| AWS_ACCESS_KEY_ID: ${{secrets.AWS_S3_ACCESS_KEY}} | |
| AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_S3_SECRET_KEY}} | |
| VCPKG_NUGET_USER: ${{secrets.VCPKG_NUGET_USER || github.repository_owner}} | |
| VCPKG_NUGET_TOKEN: ${{secrets.VCPKG_NUGET_TOKEN || secrets.GITHUB_TOKEN}} | |
| VCPKG_MAN_NUGET_USER: ${{secrets.VCPKG_MAN_NUGET_USER}} # For forks to download pre-compiled dependencies from the Man repo | |
| VCPKG_MAN_NUGET_TOKEN: ${{secrets.VCPKG_MAN_NUGET_TOKEN}} | |
| CMAKE_C_COMPILER_LAUNCHER: sccache | |
| CMAKE_CXX_COMPILER_LAUNCHER: sccache | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| defaults: | |
| run: {shell: bash} | |
| steps: | |
| - uses: actions/checkout@v3.3.0 | |
| with: | |
| lfs: 'true' | |
| fetch-depth: 0 | |
| submodules: recursive | |
| token: ${{ secrets.ARCTICDB_TEST_PAT }} | |
| - name: Configure sccache | |
| uses: mozilla-actions/sccache-action@v0.0.9 | |
| with: | |
| disable_annotations: 'true' # supress noisy report that pollutes the summary page | |
| - name: Extra envs | |
| shell: bash -l {0} | |
| run: | | |
| . build_tooling/prep_cpp_build.sh | |
| . build_tooling/vcpkg_caching.sh | |
| echo -e "VCPKG_BINARY_SOURCES=$VCPKG_BINARY_SOURCES | |
| VCPKG_ROOT=$PLATFORM_VCPKG_ROOT" | tee -a $GITHUB_ENV | |
| cmake -P cpp/CMake/CpuCount.cmake | sed 's/^-- //' | tee -a $GITHUB_ENV | |
| env: | |
| CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}} | |
| - name: Set persistent storage variables | |
| uses: ./.github/actions/set_persistent_storage_env_vars | |
| with: | |
| bucket: "arcticdb-asv-real-storage" | |
| aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}" | |
| aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}" | |
| - name: Free Disk Space (Container) | |
| run: | | |
| rm -rf /mnt/usr/local/lib/android | |
| rm -rf /mnt/usr/share/dotnet | |
| rm -rf /mnt/usr/local/share/boost | |
| rm -rf /mnt/opt/ghc | |
| rm -rf /mnt/opt/hostedtoolcache/CodeQL | |
| rm -rf /mnt/usr/local/lib/node_modules | |
| # Workaround for https://github.com/airspeed-velocity/asv/issues/1465 | |
| - name: Setup micromamba | |
| uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| micromamba-version: 2.1.0-0 | |
| - name: Install libmambapy | |
| shell: bash -el {0} | |
| run: | | |
| micromamba install -y -c conda-forge "libmambapy<2" pip python | |
| - name: Install ASV | |
| shell: bash -el {0} | |
| run: | | |
| git config --global --add safe.directory . | |
| python -m pip install --upgrade pip | |
| pip install asv | |
| asv machine --yes | |
| - name: Build project for ASV | |
| run: | | |
| python -m pip install -ve .[Testing] | |
| - name: Run ASV Tests Check script | |
| run: | | |
| python python/utils/asv_checks.py | |
| continue-on-error: false | |
| - name: Disk usage | |
| if: always() | |
| run: du -m . | sort -n | tail -n 50 || true; df -h | |
| continue-on-error: true | |
| # ========================= Sanitizers ======================== | |
| sanitizer-address-leak-undefined: | |
| name: Tests with ASan+LSan+UBSan | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: cpp-tests | |
| cmake_preset_type: debug | |
| matrix: '[{"os": "linux", "distro": "ubuntu-22.04", "container": {"image": "ghcr.io/man-group/arcticdb-dev-clang:latest"}, "cmake_preset_prefix": "linux", "cibw_build_suffix": "manylinux_x86_64", "build_dir": "/tmp/cpp_build_asan", "vcpkg_installed_dir": "/tmp/vcpkg_installed_asan", "vcpkg_packages_dir": "/tmp/vcpkg_packages_asan", "symbols": "*.pdb", "do_not_archive": ["*.o", "*.obj", "*.a", "*.lib"], "test_services": {}, "cmake_extra_args": "''-DARCTICDB_USING_ADDRESS_SANITIZER=ON'', ''-DARCTICDB_USING_UB_SANITIZER=ON''", "envs": "ARCTICDB_USE_SANITIZER=address,leak,undefined"}]' | |
| # Temporarily disabled due to too many issues that need to suppressed | |
| # sanitizer-thread: | |
| # name: Tests with TSan | |
| # uses: ./.github/workflows/build_steps.yml | |
| # secrets: inherit | |
| # permissions: {packages: write} | |
| # with: | |
| # job_type: cpp-tests | |
| # cmake_preset_type: debug | |
| # matrix: '[{"os": "linux", "distro": "ubuntu-22.04", "container": {"image": "ghcr.io/man-group/arcticdb-dev-clang:latest"}, "cmake_preset_prefix": "linux", "cibw_build_suffix": "manylinux_x86_64", "build_dir": "/tmp/cpp_build_tsan", "vcpkg_installed_dir": "/tmp/vcpkg_installed_tsan", "vcpkg_packages_dir": "/tmp/vcpkg_packages_tsan", "symbols": "*.pdb", "do_not_archive": ["*.o", "*.obj", "*.a", "*.lib"], "test_services": {}, "cmake_extra_args": "''-DARCTICDB_USING_THREAD_SANITIZER=ON''", "envs": "ARCTICDB_USE_SANITIZER=thread"}]' | |
| # code_coverage: | |
| # runs-on: "ubuntu-22.04" | |
| # container: | |
| # image: quay.io/pypa/manylinux_2_28_x86_64:latest | |
| # services: | |
| # mongodb: | |
| # image: mongo:4.4 | |
| # ports: | |
| # - 27017:27017 | |
| # env: | |
| # VCPKG_NUGET_USER: ${{secrets.VCPKG_NUGET_USER || github.repository_owner}} | |
| # VCPKG_NUGET_TOKEN: ${{secrets.VCPKG_NUGET_TOKEN || secrets.GITHUB_TOKEN}} | |
| # VCPKG_MAN_NUGET_USER: ${{secrets.VCPKG_MAN_NUGET_USER}} # For forks to download pre-compiled dependencies from the Man repo | |
| # VCPKG_MAN_NUGET_TOKEN: ${{secrets.VCPKG_MAN_NUGET_TOKEN}} | |
| # ARCTIC_CMAKE_PRESET: linux-debug | |
| # ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| # steps: | |
| # - uses: actions/checkout@v3.3.0 | |
| # with: | |
| # submodules: recursive | |
| # - name: Get number of CPU cores | |
| # uses: SimenB/github-actions-cpu-cores@v1.1.0 | |
| # id: cpu-cores | |
| # - name: Install deps | |
| # uses: ./.github/actions/setup_deps | |
| # - name: Extra envs | |
| # shell: bash -l {0} | |
| # run: | | |
| # . build_tooling/vcpkg_caching.sh # Linux follower needs another call in CIBW | |
| # echo -e "VCPKG_BINARY_SOURCES=$VCPKG_BINARY_SOURCES | |
| # VCPKG_ROOT=$PLATFORM_VCPKG_ROOT" | tee -a $GITHUB_ENV | |
| # cmake -P cpp/CMake/CpuCount.cmake | sed 's/^-- //' | tee -a $GITHUB_ENV | |
| # echo "ARCTICDB_CODE_COVERAGE_BUILD=1" | tee -a $GITHUB_ENV | |
| # env: | |
| # CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}} | |
| # - name: Prepare C++ compilation env | |
| # run: . build_tooling/prep_cpp_build.sh | |
| # - name: CMake compile | |
| # # We are pinning the version to 10.6 because >= 10.7, use node20 which is not supported in the container | |
| # uses: lukka/run-cmake@v10.6 | |
| # with: | |
| # cmakeListsTxtPath: ${{github.workspace}}/cpp/CMakeLists.txt | |
| # configurePreset: ${{env.ARCTIC_CMAKE_PRESET}} | |
| # buildPreset: ${{env.ARCTIC_CMAKE_PRESET}} | |
| # env: | |
| # ARCTICDB_DEBUG_FIND_PYTHON: ${{vars.ARCTICDB_DEBUG_FIND_PYTHON}} | |
| # python_impl_name: 'cp311' | |
| # - name: Run C++ Tests | |
| # shell: bash -l {0} | |
| # run: | | |
| # cd cpp/out/linux-debug-build/ | |
| # ls arcticdb | |
| # make -j ${{ steps.cpu-cores.outputs.count }} arcticdb_rapidcheck_tests | |
| # make -j ${{ steps.cpu-cores.outputs.count }} test_unit_arcticdb | |
| # ctest | |
| # # We are chainging the python here because we want to use the default python to build (it is devel version) | |
| # # and this python for the rest of the testing | |
| # - name: Select Python (Linux) | |
| # run: echo /opt/python/cp36-cp36m/bin >> $GITHUB_PATH | |
| # - name: Install local dependencies with pip | |
| # shell: bash | |
| # run: | | |
| # python -m pip install --upgrade pip | |
| # ARCTIC_CMAKE_PRESET=skip pip install -ve .[Testing] | |
| # # - name: Test with pytest | |
| # # uses: ./.github/actions/run_local_pytest | |
| # # with: | |
| # # build_type: debug | |
| # # threads: 1 | |
| # # fast_tests_only: 0 | |
| # # other_params: '-m coverage run ' | |
| # - name: Get python Coverage report | |
| # shell: bash -l {0} | |
| # run: | | |
| # cd python | |
| # python -m coverage report -m | tee output.txt | |
| # python -m coverage html | |
| # zip -r python_cov.zip htmlcov/ | |
| # echo "PYTHON_COV_PERCENT=$(cat output.txt | grep 'TOTAL' | awk '{print $NF}' | tr -d '%')" >> $GITHUB_ENV | |
| # - name: Run Gcovr manually post-pytest | |
| # shell: bash -l {0} | |
| # run: | | |
| # cd cpp/out/linux-debug-build/ | |
| # python -m pip install gcovr | |
| # mkdir coverage | |
| # python -m gcovr --txt --html-details coverage/index.html -e vcpkg_installed/ -e proto/ -e ../../third_party -e ../../arcticdb/util/test/ -r ../.. --exclude-throw-branches --exclude-unreachable-branches -u --exclude-function-lines | tee output.txt | |
| # zip -r coverage.zip coverage/ | |
| # echo "CPP_COV_PERCENT=$(cat output.txt | grep 'TOTAL' | awk '{print $NF}' | tr -d '%')" >> $GITHUB_ENV | |
| # - name: Upload Coverage | |
| # uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: cpp-coverage-artifact | |
| # path: cpp/out/linux-debug-build/coverage.zip | |
| # - name: Upload Python Coverage | |
| # uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: python-coverage-artifact | |
| # path: python/python_cov.zip | |
| # - name: Restore cached CPP Coverage Percentage from the previous run | |
| # id: cache-cov-restore | |
| # uses: actions/cache/restore@v3.3.2 | |
| # with: | |
| # path: prev_coverage.txt | |
| # key: coverage | |
| # - name: Get and compare coverage if cache was restored | |
| # run: | | |
| # # if cache was restored, compare coverage | |
| # if [ -f coverage.txt ]; then | |
| # PREV_COVERAGE=$(cat prev_coverage.txt | cut -d' ' -f2) | |
| # echo "Previous coverage: $PREV_COVERAGE" | |
| # CURR_COVERAGE=${{env.CPP_COV_PERCENT}} | |
| # echo "CPP_COV_PREV_PERCENT=$PREV_COVERAGE" >> $GITHUB_ENV | |
| # echo "Current coverage: $CURR_COVERAGE" | |
| # if [ $CURR_COVERAGE -gt $PREV_COVERAGE ]; then | |
| # echo "Coverage increased" | |
| # elif [ $CURR_COVERAGE -lt $PREV_COVERAGE ]; then | |
| # echo "Coverage decreased" | |
| # else | |
| # echo "Coverage unchanged" | |
| # fi | |
| # fi | |
| # - name: Save CPP Coverage Percentage to file | |
| # run: | | |
| # echo "Coverage: ${{ env.CPP_COV_PERCENT }}" > current_coverage.txt | |
| # - name: Save the current CPP Coverage Percentage to the cache | |
| # id: cache-cov-save | |
| # uses: actions/cache/save@v3.3.2 | |
| # with: | |
| # path: current_coverage.txt | |
| # key: coverage | |
| # - name: Check percentage and send Slack notification | |
| # if: ${{ env.CPP_COV_PREV_PERCENT && env.CPP_COV_PERCENT && env.CPP_COV_PERCENT < env.CPP_COV_PREV_PERCENT }} | |
| # uses: slackapi/slack-github-action@v1.24.0 | |
| # with: | |
| # # For posting a rich message using Block Kit | |
| # payload: | | |
| # { | |
| # "text": "The CPP Code Coverage has been reduced", | |
| # "blocks": [ | |
| # { | |
| # "type": "section", | |
| # "text": { | |
| # "type": "mrkdwn", | |
| # "text": "The CPP Code Coverage from the current run(${{ env.CPP_COV_PERCENT }}%) is lower the previous one(${{ env.CPP_COV_PREV_PERCENT }}%)." | |
| # } | |
| # } | |
| # ] | |
| # } | |
| # env: | |
| # SLACK_WEBHOOK_URL: ${{ secrets.ARCTICDB_DEV_WEBHOOK_URL }} | |
| # SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |