Add --excludes to run_benchmarks.rb to easily exclude benchmarks

Author: eregon

lib/argument_parser.rb

@@ -11,6 +11,7 @@ class ArgumentParser
     :yjit_opts,
     :categories,
     :name_filters,
+    :excludes,
     :rss,
     :graph,
     :no_pinning,
@@ -82,6 +83,10 @@ def parse(argv)
         args.name_filters = list
       end
 
+      opts.on("--excludes=x,y,z", Array, "excludes the listed benchmarks") do |list|
+        args.excludes = list
+      end
+
       opts.on("--skip-yjit", "Don't run with yjit after interpreter") do
         args.skip_yjit = true
       end
@@ -174,6 +179,7 @@ def default_args
       yjit_opts: "",
       categories: [],
       name_filters: [],
+      excludes: [],
       rss: false,
       graph: false,
       no_pinning: false,

lib/benchmark_filter.rb

@@ -2,16 +2,17 @@
 
 # Filters benchmarks based on categories and name patterns
 class BenchmarkFilter
-  def initialize(categories:, name_filters:, metadata:)
+  def initialize(categories:, name_filters:, excludes:, metadata:)
     @categories = categories
     @name_filters = process_name_filters(name_filters)
+    @excludes = excludes
     @metadata = metadata
     @category_cache = {}
   end
 
   def match?(entry)
     name = entry.sub(/\.rb\z/, '')
-    matches_category?(name) && matches_name_filter?(name)
+    matches_category?(name) && matches_name_filter?(name) && !matches_excludes?(name)
   end
 
   private
@@ -29,6 +30,10 @@ def matches_name_filter?(name)
     @name_filters.any? { |filter| filter === name }
   end
 
+  def matches_excludes?(name)
+    @excludes.include?(name)
+  end
+
   def get_benchmark_categories(name)
     @category_cache[name] ||= begin
       benchmark_metadata = @metadata[name] || {}

lib/benchmark_runner/cli.rb

@@ -40,6 +40,7 @@ def run
           ruby_description: ruby_descriptions[name],
           categories: args.categories,
           name_filters: args.name_filters,
+          excludes: args.excludes,
           out_path: args.out_path,
           harness: args.harness,
           pre_init: args.with_pre_init,

lib/benchmark_suite.rb

@@ -18,13 +18,14 @@ class BenchmarkSuite
   RACTOR_CATEGORY = ["ractor"].freeze
   RACTOR_HARNESS = "harness-ractor"
 
-  attr_reader :ruby, :ruby_description, :categories, :name_filters, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir
+  attr_reader :ruby, :ruby_description, :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir
 
-  def initialize(ruby:, ruby_description:, categories:, name_filters:, out_path:, harness:, pre_init: nil, no_pinning: false)
+  def initialize(ruby:, ruby_description:, categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false)
     @ruby = ruby
     @ruby_description = ruby_description
     @categories = categories
     @name_filters = name_filters
+    @excludes = excludes
     @out_path = out_path
     @harness = harness
     @pre_init = pre_init ? expand_pre_init(pre_init) : nil
@@ -148,6 +149,7 @@ def main_benchmark_filter
     @main_benchmark_filter ||= BenchmarkFilter.new(
       categories: categories,
       name_filters: name_filters,
+      excludes: excludes,
       metadata: benchmarks_metadata
     )
   end
@@ -156,6 +158,7 @@ def ractor_benchmark_filter
     @ractor_benchmark_filter ||= BenchmarkFilter.new(
       categories: [],
       name_filters: name_filters,
+      excludes: excludes,
       metadata: benchmarks_metadata
     )
   end

test/benchmark_filter_test.rb

@@ -14,73 +14,80 @@
 
   describe '#match?' do
     it 'matches when no filters provided' do
-      filter = BenchmarkFilter.new(categories: [], name_filters: [], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: [], name_filters: [], excludes: [], metadata: @metadata)
 
       assert_equal true, filter.match?('fib.rb')
     end
 
     it 'matches by category' do
-      filter = BenchmarkFilter.new(categories: ['micro'], name_filters: [], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: ['micro'], name_filters: [], excludes: [], metadata: @metadata)
 
       assert_equal true, filter.match?('fib.rb')
       assert_equal false, filter.match?('railsbench.rb')
     end
 
     it 'matches by name filter' do
-      filter = BenchmarkFilter.new(categories: [], name_filters: ['fib'], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: [], name_filters: ['fib'], excludes: [], metadata: @metadata)
 
       assert_equal true, filter.match?('fib.rb')
       assert_equal false, filter.match?('railsbench.rb')
     end
 
+    it 'applies excludes' do
+      filter = BenchmarkFilter.new(categories: ['headline'], name_filters: [], excludes: ['railsbench'], metadata: @metadata)
+
+      assert_equal true, filter.match?('optcarrot.rb')
+      assert_equal false, filter.match?('railsbench.rb')
+    end
+
     it 'matches ractor category' do
-      filter = BenchmarkFilter.new(categories: ['ractor'], name_filters: [], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: ['ractor'], name_filters: [], excludes: [], metadata: @metadata)
 
       assert_equal true, filter.match?('ractor_bench.rb')
     end
 
     it 'strips .rb extension from entry name' do
-      filter = BenchmarkFilter.new(categories: [], name_filters: ['fib'], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: [], name_filters: ['fib'], excludes: [], metadata: @metadata)
 
       assert_equal true, filter.match?('fib.rb')
     end
 
     it 'handles regex filters' do
-      filter = BenchmarkFilter.new(categories: [], name_filters: ['/rails/'], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: [], name_filters: ['/rails/'], excludes: [], metadata: @metadata)
 
       assert_equal true, filter.match?('railsbench.rb')
       assert_equal false, filter.match?('fib.rb')
     end
 
     it 'handles case-insensitive regex filters' do
-      filter = BenchmarkFilter.new(categories: [], name_filters: ['/RAILS/i'], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: [], name_filters: ['/RAILS/i'], excludes: [], metadata: @metadata)
 
       assert_equal true, filter.match?('railsbench.rb')
     end
 
     it 'handles multiple categories' do
-      filter = BenchmarkFilter.new(categories: ['micro', 'headline'], name_filters: [], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: ['micro', 'headline'], name_filters: [], excludes: [], metadata: @metadata)
 
       assert_equal true, filter.match?('fib.rb')
       assert_equal true, filter.match?('railsbench.rb')
     end
 
     it 'requires both category and name filter to match when both provided' do
-      filter = BenchmarkFilter.new(categories: ['micro'], name_filters: ['rails'], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: ['micro'], name_filters: ['rails'], excludes: [], metadata: @metadata)
 
       assert_equal false, filter.match?('fib.rb') # matches category but not name
       assert_equal false, filter.match?('railsbench.rb') # matches name but not category
     end
 
     it 'handles complex regex patterns' do
-      filter = BenchmarkFilter.new(categories: [], name_filters: ['/opt.*rot/'], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: [], name_filters: ['/opt.*rot/'], excludes: [], metadata: @metadata)
 
       assert_equal true, filter.match?('optcarrot.rb')
       assert_equal false, filter.match?('fib.rb')
     end
 
     it 'handles mixed string and regex filters' do
-      filter = BenchmarkFilter.new(categories: [], name_filters: ['fib', '/rails/'], metadata: @metadata)
+      filter = BenchmarkFilter.new(categories: [], name_filters: ['fib', '/rails/'], excludes: [], metadata: @metadata)
 
       assert_equal true, filter.match?('fib.rb')
       assert_equal true, filter.match?('railsbench.rb')

test/benchmark_runner_cli_test.rb

@@ -41,6 +41,7 @@ def create_args(overrides = {})
       yjit_opts: '',
       categories: [],
       name_filters: [],
+      excludes: [],
       rss: false,
       graph: false,
       no_pinning: true,