Separate harness implementation from the harness framework #1516
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test benchmarks | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| schedule: | |
| - cron: "00 15 * * *" | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ruby: [ruby, head, truffleruby] | |
| if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Cache apt packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| /var/cache/apt/archives | |
| /var/lib/apt/lists | |
| key: ${{ runner.os }}-apt-imagemagick-${{ hashFiles('.github/workflows/test.yml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-apt-imagemagick- | |
| - name: Install ImageMagick dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends libmagickwand-dev | |
| - name: Set up Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: ${{ matrix.ruby }} | |
| - name: Run tests | |
| run: rake test | |
| continue-on-error: ${{ matrix.ruby == 'truffleruby' }} | |
| benchmark-default: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ruby: [ruby, head, truffleruby] | |
| if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: ${{ matrix.ruby }} | |
| - name: Test run_benchmarks.rb | |
| run: ./run_benchmarks.rb | |
| env: | |
| WARMUP_ITRS: '1' | |
| MIN_BENCH_ITRS: '1' | |
| MIN_BENCH_TIME: '0' | |
| continue-on-error: ${{ matrix.ruby == 'truffleruby' }} | |
| benchmark-ractor: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ruby: [ruby, head] | |
| if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: ${{ matrix.ruby }} | |
| - name: Test run_benchmarks.rb - Ractors | |
| run: ./run_benchmarks.rb --category=ractor | |
| env: | |
| WARMUP_ITRS: '1' | |
| MIN_BENCH_ITRS: '1' | |
| MIN_BENCH_TIME: '0' | |
| benchmark-ractor-only: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ruby: [ruby, head] | |
| if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: ${{ matrix.ruby }} | |
| - name: Test run_benchmarks.rb - Ractor Only | |
| run: ./run_benchmarks.rb --category=ractor-only | |
| env: | |
| WARMUP_ITRS: '1' | |
| MIN_BENCH_ITRS: '1' | |
| MIN_BENCH_TIME: '0' | |
| benchmark-graph: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Cache apt packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| /var/cache/apt/archives | |
| /var/lib/apt/lists | |
| key: ${{ runner.os }}-apt-imagemagick-${{ hashFiles('.github/workflows/test.yml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-apt-imagemagick- | |
| - name: Install ImageMagick dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends libmagickwand-dev | |
| - name: Set up Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: ruby | |
| - name: Test run_benchmarks.rb --graph | |
| run: ./run_benchmarks.rb --graph fib | |
| env: | |
| WARMUP_ITRS: '1' | |
| MIN_BENCH_ITRS: '1' | |
| MIN_BENCH_TIME: '0' | |
| benchmark-run-once: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: head | |
| - name: Test run_once.sh | |
| run: ./run_once.sh --yjit-stats benchmarks/railsbench/benchmark.rb | |
| all-tests: | |
| runs-on: ubuntu-latest | |
| needs: [test, benchmark-default, benchmark-ractor, benchmark-ractor-only, benchmark-graph, benchmark-run-once] | |
| if: always() | |
| steps: | |
| - name: Check all job results | |
| run: | | |
| if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then | |
| echo "One or more jobs failed" | |
| exit 1 | |
| elif [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then | |
| echo "One or more jobs were cancelled" | |
| exit 1 | |
| else | |
| echo "All jobs completed successfully" | |
| fi |