From e51b4bd98ef1978751a2f5ce44f61b18d4f5f2ba Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 14:05:50 +0100 Subject: [PATCH 01/10] Share the Ractor count list between the harness and the runner In this commit we move the default Ractor counts and the RUBY_BENCH_RACTORS parsing into lib/ractor_counts.rb, so that run_benchmarks.rb can see the same list of counts as harness-ractor. --- harness-ractor/harness.rb | 14 ++------------ lib/ractor_counts.rb | 13 +++++++++++++ 2 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 lib/ractor_counts.rb diff --git a/harness-ractor/harness.rb b/harness-ractor/harness.rb index f7c42667..afd6b9ca 100644 --- a/harness-ractor/harness.rb +++ b/harness-ractor/harness.rb @@ -17,18 +17,8 @@ {}.freeze end -default_ractors = [ - 0, # without ractor - 1, 2, 4, 6, 8#, 12, 16, 32 -] -if rs = ENV["RUBY_BENCH_RACTORS"] - rs = rs.split(",").map(&:to_i) # If you want to include 0, you have to specify - rs = rs.sort.uniq - if rs.any? - ractors = rs - end -end -RACTORS = (ractors || default_ractors).freeze +require_relative '../lib/ractor_counts' +RACTORS = RactorCounts.from_env unless Ractor.method_defined?(:join) class Ractor diff --git a/lib/ractor_counts.rb b/lib/ractor_counts.rb new file mode 100644 index 00000000..15aada1b --- /dev/null +++ b/lib/ractor_counts.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module RactorCounts + DEFAULT = [0, 1, 2, 4, 6, 8].freeze + ENV_VAR = "RUBY_BENCH_RACTORS" + + module_function + + def from_env(env = ENV) + counts = env.fetch(ENV_VAR, "").split(",").map(&:to_i).sort.uniq + counts.empty? ? DEFAULT : counts.freeze + end +end From fc6e9a417290a2278490ead1211d9b3128fa0741 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 14:06:28 +0100 Subject: [PATCH 02/10] Warm up at each measured Ractor count The harness ran every warmup iteration at r=0 in the main Ractor, and then measured each count straight after the previous one. So no count above 0 was ever warmed up at its own count. In this commit each count runs WARMUP_ITRS iterations at that count before it measures, in both timing and --ractor-gc mode. --- harness-ractor/harness.rb | 60 +++++++++++++++++-------------------- test/ractor_harness_test.rb | 37 +++++++++++++++++++++++ 2 files changed, 64 insertions(+), 33 deletions(-) create mode 100644 test/ractor_harness_test.rb diff --git a/harness-ractor/harness.rb b/harness-ractor/harness.rb index afd6b9ca..c34975f1 100644 --- a/harness-ractor/harness.rb +++ b/harness-ractor/harness.rb @@ -41,13 +41,11 @@ def run_benchmark(num_itrs_hint, ractor_args: [], &block) check_ractor_gc_support gc_config = GC.config.transform_keys(&:to_s) if GC.respond_to?(:config) GCStats.with_measure_total_time do - run_warmup(warmup_itrs, ractor_args, &block) - run_benchmark_gc(bench_itrs, Ractor.make_shareable(block), ractor_args, gc_config: gc_config) + run_benchmark_gc(warmup_itrs, bench_itrs, Ractor.make_shareable(block), ractor_args, gc_config: gc_config) end else puts "r: itr: time" - run_warmup(warmup_itrs, ractor_args, &block) - run_benchmark_timing(bench_itrs, ractor_args, &block) + run_benchmark_timing(warmup_itrs, bench_itrs, ractor_args, &block) end end @@ -63,43 +61,37 @@ def check_ractor_gc_support end end -def run_warmup(warmup_itrs, ractor_args, &block) - warmup_itrs.times do - args = ractor_args.empty? ? [] : ractor_deep_dup(ractor_args) - block.call(*([0] + args)) - end -end - -def run_benchmark_timing(bench_itrs, ractor_args, &block) +def run_benchmark_timing(warmup_itrs, bench_itrs, ractor_args, &block) stats = Hash.new { |h,k| h[k] = [] } RACTORS.each do |rs| - num_itrs = 0 - while num_itrs < bench_itrs - before = Process.clock_gettime(Process::CLOCK_MONOTONIC) - if rs.zero? - block.call *([rs] + ractor_deep_dup(ractor_args)) - else - rs_list = [] - rs.times do - rs_list << Ractor.new(*([rs] + ractor_args), &block) # ractor_args are copied - end - while rs_list.any? - r, _obj = Ractor.select(*rs_list) - rs_list.delete(r) - end - end - num_itrs += 1 - time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - before - time_ms = (1000 * time).to_i - itr_str = "%-3s %4s %6s" % ["#{rs}", "##{num_itrs}:", "#{time_ms}ms"] + warmup_itrs.times { run_timing_iteration(rs, ractor_args, &block) } + bench_itrs.times do |i| + time = run_timing_iteration(rs, ractor_args, &block) stats[rs] << time - puts itr_str + puts "%-3s %4s %6s" % ["#{rs}", "##{i + 1}:", "#{(1000 * time).to_i}ms"] end end return_results([], stats.values.flatten, bench_by_ractors: stats) end +def run_timing_iteration(rs, ractor_args, &block) + before = Process.clock_gettime(Process::CLOCK_MONOTONIC) + if rs.zero? + block.call *([rs] + ractor_deep_dup(ractor_args)) + else + rs_list = [] + rs.times do + rs_list << Ractor.new(*([rs] + ractor_args), &block) # ractor_args are copied + end + while rs_list.any? + r, _obj = Ractor.select(*rs_list) + rs_list.delete(r) + end + end + Process.clock_gettime(Process::CLOCK_MONOTONIC) - before +end + RACTOR_GC_SERIES = { "gc_count_bench" => "gc_count", "gc_global_count_bench" => "gc_global_count", @@ -110,7 +102,7 @@ def run_benchmark_timing(bench_itrs, ractor_args, &block) "gc_total_time_bench" => "gc_total_time_ns", }.freeze -def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) +def run_benchmark_gc(warmup_itrs, bench_itrs, block, ractor_args, gc_config:) stats = Hash.new { |h,k| h[k] = [] } gc_by_ractors = {} @@ -124,6 +116,8 @@ def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) group["gc_controller_samples"] = [] if rs > 0 series = Hash.new { |h,k| h[k] = [] } + warmup_itrs.times { run_ractor_gc_iteration(rs, ractor_args, &block) } + num_itrs = 0 while num_itrs < bench_itrs num_itrs += 1 diff --git a/test/ractor_harness_test.rb b/test/ractor_harness_test.rb new file mode 100644 index 00000000..a1c12207 --- /dev/null +++ b/test/ractor_harness_test.rb @@ -0,0 +1,37 @@ +require_relative 'test_helper' +require 'json' +require 'open3' +require 'tmpdir' + +describe 'Ractor harness' do + it 'warms up at the measured Ractor count' do + skip 'target Ruby has no Ractor' unless defined?(Ractor) + root = File.expand_path('..', __dir__) + Dir.mktmpdir do |dir| + result_path = File.join(dir, 'results.json') + calls_path = File.join(dir, 'calls.txt') + script = File.join(dir, 'workload.rb') + File.write(script, <<~RUBY) + require #{File.join(root, 'harness', 'loader').inspect} + run_benchmark(1) do |count| + File.open(#{calls_path.inspect}, 'a') { |f| f.puts(count) } + end + RUBY + env = { + 'RUBYOPT' => nil, + 'RUBYLIB' => nil, + 'BUNDLE_GEMFILE' => nil, + 'RUBY_BENCH_RACTORS' => '2', + 'WARMUP_ITRS' => '2', + 'MIN_BENCH_ITRS' => '1', + 'RESULT_JSON_PATH' => result_path + } + + stdout, stderr, status = Open3.capture3(env, RbConfig.ruby, "-I#{File.join(root, 'harness-ractor')}", script) + + assert status.success?, "workload failed:\n#{stdout}\n#{stderr}" + assert_equal ['2'], JSON.parse(File.read(result_path))['bench_by_ractors'].keys + assert_equal ['2'] * 6, File.readlines(calls_path, chomp: true), '(2 warmup + 1 measured) iterations x 2 Ractors' + end + end +end From 5c5960d79c4609b4cb97507ac96484cc6f6e0228 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 14:07:01 +0100 Subject: [PATCH 03/10] Keep non-Ractor rows in the Ractor row layout RactorRowLayout dropped every benchmark that has no bench_by_ractors breakdown, so normal benchmarks vanished from a table that mixes them with Ractor benchmarks. --- lib/row_layout.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/row_layout.rb b/lib/row_layout.rb index 92b47e5d..159fce63 100644 --- a/lib/row_layout.rb +++ b/lib/row_layout.rb @@ -32,7 +32,7 @@ def entries(bench_names) seen[base_name] = true members = @groups_by_base_name[base_name] - next [] unless members + next [Entry.new(data_key: data_key, label_cells: [data_key, ''])] unless members members.each_with_index.map do |(member_key, count), i| name_cell = i.zero? ? base_name : '' From f2640aa192c0e781ca5c258629896db655c367bc Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 14:07:20 +0100 Subject: [PATCH 04/10] Merge per-count Ractor results into one blob In this commit we add RactorBreakdown.merge, which combines one result blob per Ractor count into the existing bench_by_ractors and gc_by_ractors shape. Process-level data (rss, maxrss, JIT stats and command_line) stays per count under results_by_ractors and is never promoted to the top level. expand copies it into each count's row, rather than repeating the values of one process for every count. --- lib/ractor_breakdown.rb | 28 +++++++++++++++++++- test/ractor_breakdown_test.rb | 48 +++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/lib/ractor_breakdown.rb b/lib/ractor_breakdown.rb index 9a849595..1f318819 100644 --- a/lib/ractor_breakdown.rb +++ b/lib/ractor_breakdown.rb @@ -2,6 +2,8 @@ module RactorBreakdown KEY_SEP = "\x00" + MEASUREMENT_KEYS = %w[warmup bench bench_by_ractors gc_by_ractors].freeze + PROCESS_KEYS = %w[rss maxrss yjit_stats zjit_stats zjit_stats_string command_line].freeze Result = Struct.new(:bench_data, :groups) @@ -15,6 +17,28 @@ def base_name(data_key) data_key.split(KEY_SEP, 2).first end + def merge(blobs_by_count) + counts = blobs_by_count.keys.sort + process_data = counts.to_h do |count| + [count.to_s, blobs_by_count[count].reject { |k, _| MEASUREMENT_KEYS.include?(k) }] + end + first, *rest = process_data.values + merged = first.select do |k, v| + !PROCESS_KEYS.include?(k) && rest.all? { |data| data.key?(k) && data[k] == v } + end + + merged['warmup'] = [] + merged['bench'] = counts.flat_map { |count| blobs_by_count[count]['bench'] } + merged['bench_by_ractors'] = counts.to_h { |count| [count.to_s, blobs_by_count[count]['bench']] } + gc_by_ractors = counts.filter_map do |count| + group = blobs_by_count[count].dig('gc_by_ractors', count.to_s) + [count.to_s, group] if group + end.to_h + merged['gc_by_ractors'] = gc_by_ractors unless gc_by_ractors.empty? + merged['results_by_ractors'] = process_data.transform_values { |data| data.reject { |k, _| merged.key?(k) } } + merged + end + def expand(bench_data) groups = {} new_data = {} @@ -42,7 +66,9 @@ def expand(bench_data) end def per_count_blob(blob, breakdown, count) - per_count = blob.reject { |k, _| k == 'bench_by_ractors' || k == 'gc_by_ractors' || k == 'bench' } + per_count = blob.reject { |k, _| k == 'bench_by_ractors' || k == 'gc_by_ractors' || k == 'results_by_ractors' || k == 'bench' } + process_data = blob['results_by_ractors'] + per_count.merge!(process_data[count.to_s]) if process_data.is_a?(Hash) && process_data.key?(count.to_s) per_count['bench'] = breakdown[count.to_s] per_count['warmup'] = [] gc_by_ractors = blob['gc_by_ractors'] diff --git a/test/ractor_breakdown_test.rb b/test/ractor_breakdown_test.rb index 11e6335d..7f130189 100644 --- a/test/ractor_breakdown_test.rb +++ b/test/ractor_breakdown_test.rb @@ -165,4 +165,52 @@ assert_equal({ 'implementation' => 'default' }, per_count['gc_config']) end end + + describe '.merge' do + def child_blob(count, bench:, rss:, zjit_calls:) + { + 'RUBY_DESCRIPTION' => 'ruby 4.1.0', + 'warmup' => [], + 'bench' => bench, + 'bench_by_ractors' => { count.to_s => bench }, + 'gc_scope' => 'ractor-local-workload', + 'gc_by_ractors' => { count.to_s => { 'gc_count_bench' => [count * 10] } }, + 'rss' => rss, + 'maxrss' => 500, + 'zjit_stats' => { 'calls' => zjit_calls }, + 'command_line' => "RUBY_BENCH_RACTORS=#{count} ruby bench.rb" + } + end + + it 'keeps process-level data per count so each expanded row shows its own process' do + merged = RactorBreakdown.merge( + 2 => child_blob(2, bench: [2.0, 2.1], rss: 300, zjit_calls: 7), + 0 => child_blob(0, bench: [1.0, 1.1], rss: 100, zjit_calls: 5) + ) + + assert_equal({ '0' => [1.0, 1.1], '2' => [2.0, 2.1] }, merged['bench_by_ractors']) + assert_equal [1.0, 1.1, 2.0, 2.1], merged['bench'] + refute merged.key?('rss') + refute merged.key?('maxrss'), 'equal process-level values must stay per count' + assert_equal 'ruby 4.1.0', merged['RUBY_DESCRIPTION'] + assert_equal 'ractor-local-workload', merged['gc_scope'] + refute merged.key?('zjit_stats') + refute merged.key?('command_line') + assert_equal %w[rss maxrss zjit_stats command_line], merged['results_by_ractors']['0'].keys + + exe = RactorBreakdown.expand({ 'ruby' => { 'r' => merged } }).bench_data['ruby'] + r0 = exe["r\x000"] + r2 = exe["r\x002"] + + assert_equal [1.0, 1.1], r0['bench'] + assert_equal 100, r0['rss'] + assert_equal 300, r2['rss'] + assert_equal({ 'calls' => 5 }, r0['zjit_stats']) + assert_equal({ 'calls' => 7 }, r2['zjit_stats']) + assert_equal [0], r0['gc_count_bench'] + assert_equal [20], r2['gc_count_bench'] + assert_equal 'RUBY_BENCH_RACTORS=2 ruby bench.rb', r2['command_line'] + refute r0.key?('results_by_ractors') + end + end end From aa115a323dc9250dde51208ca42c401fc3998a90 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 14:07:42 +0100 Subject: [PATCH 05/10] Write one CSV row per Ractor count The CSV was built from the raw blobs, so a Ractor benchmark was a single row with the timings of every count flattened together. It now uses the same per-count breakdown as the text summary. --- lib/benchmark_runner/cli.rb | 13 +++++++++++-- test/benchmark_runner_cli_test.rb | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index ed59d2d1..da1a73c2 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -110,12 +110,14 @@ def run puts # Build the results table + csv_data, csv_layout = csv_view(bench_data) builder = ResultsTableBuilder.new( executable_names: ruby_descriptions.keys, - bench_data: bench_data, + bench_data: csv_data, include_rss: args.rss, include_pvalue: args.pvalue, - zjit_stats: args.zjit_stats + zjit_stats: args.zjit_stats, + row_layout: csv_layout ) table, format, gc_table, gc_format = builder.build @@ -158,6 +160,13 @@ def run private + def csv_view(bench_data) + breakdown = RactorBreakdown.expand(bench_data) + return [bench_data, FlatRowLayout.new] if breakdown.groups.empty? + + [breakdown.bench_data, RactorRowLayout.new(groups: breakdown.groups)] + end + def build_output_sections(executable_names, bench_data, bench_harnesses, bench_failures) ordered_names = sorted_benchmark_names(executable_names, bench_data) failed_names = bench_failures.values.flat_map(&:keys).uniq diff --git a/test/benchmark_runner_cli_test.rb b/test/benchmark_runner_cli_test.rb index 7baf82f3..1908ac90 100644 --- a/test/benchmark_runner_cli_test.rb +++ b/test/benchmark_runner_cli_test.rb @@ -197,6 +197,25 @@ def create_args(overrides = {}) end end + describe '#csv_view' do + it 'gives each Ractor count its own CSV row with that process RSS' do + cli = BenchmarkRunner::CLI.new(create_args) + mib = 1024 * 1024 + merged = RactorBreakdown.merge( + 0 => { 'warmup' => [], 'bench' => [1.0], 'rss' => 10 * mib }, + 2 => { 'warmup' => [], 'bench' => [2.0], 'rss' => 30 * mib } + ) + bench_data = { 'ruby' => { 'fib' => { 'warmup' => [], 'bench' => [0.1], 'rss' => 5 * mib }, 'gvl' => merged } } + + data, layout = cli.send(:csv_view, bench_data) + table, = ResultsTableBuilder.new(executable_names: ['ruby'], bench_data: data, include_rss: true, row_layout: layout).build + + assert_equal ['bench', 'ractors', 'ruby (ms)', 'RSS (MiB)'], table[0] + rows = table.drop(1).to_h { |row| [row[0..1], row[3]] } + assert_equal({ ['fib', ''] => 5.0, ['gvl', '0'] => 10.0, ['', '2'] => 30.0 }, rows) + end + end + describe '#run integration test' do it 'runs a simple benchmark end-to-end and produces all output files' do Dir.mktmpdir do |tmpdir| From 929dedf104006b04ce040221f93bcdf25875dbcd Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 14:07:30 +0100 Subject: [PATCH 06/10] Run each Ractor count in its own process All the Ractor counts ran one after another in a single process. The main objspace, the global page pool and the JIT state carried over from one count into the next, so the r=4 result was really "r=4 after r=0, 1 and 2". In this commit run_benchmarks.rb starts a fresh process per count with RUBY_BENCH_RACTORS=, gives each one its own temp result file, and merges them. If any count fails, the whole benchmark fails. --- lib/benchmark_suite.rb | 54 ++++++++++++++++++++++++------- test/benchmark_suite_test.rb | 63 ++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 11 deletions(-) diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 4c22a12a..c387e770 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -10,6 +10,8 @@ require_relative 'benchmark_filter' require_relative 'benchmark_runner' require_relative 'benchmark_discovery' +require_relative 'ractor_breakdown' +require_relative 'ractor_counts' # BenchmarkSuite runs a collection of benchmarks and collects their results class BenchmarkSuite @@ -48,22 +50,16 @@ def run_benchmark(entry, ruby:, ruby_description:) env = benchmark_env(ruby) caller_json_path = ENV["RESULT_JSON_PATH"] quiet = ENV['BENCHMARK_QUIET'] == '1' - - result_json_path = caller_json_path || File.join(out_path, "temp#{Process.pid}.json") cmd_prefix = base_cmd(ruby_description, entry.name) - - # Clear project-level Bundler environment so benchmarks run in a clean context. - # Benchmarks that need Bundler (e.g., railsbench) set up their own via use_gemfile. benchmark_harness = benchmark_harness_for(entry.name) - result = if defined?(Bundler) - Bundler.with_unbundled_env do - run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet) - end - else - run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet) + if benchmark_harness == RACTOR_HARNESS + return run_ractor_benchmark(entry, ruby, cmd_prefix, env, benchmark_harness, caller_json_path, quiet: quiet) end + result_json_path = caller_json_path || File.join(out_path, "temp#{Process.pid}.json") + result = run_benchmark_process(entry.script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet) + if result[:success] { name: entry.name, data: process_benchmark_result(result_json_path, result[:command], delete_file: !caller_json_path), harness: benchmark_harness } else @@ -106,6 +102,42 @@ def setup_benchmark_directories end end + def run_ractor_benchmark(entry, ruby, cmd_prefix, env, benchmark_harness, caller_json_path, quiet: false) + blobs_by_count = {} + + RactorCounts.from_env.each do |count| + result_json_path = File.join(out_path, "temp#{Process.pid}_r#{count}.json") + count_env = env.merge(RactorCounts::ENV_VAR => count.to_s) + result = run_benchmark_process(entry.script_path, result_json_path, ruby, cmd_prefix, count_env, benchmark_harness, quiet: quiet) + + unless result[:success] + FileUtils.rm_f(result_json_path) + return { name: entry.name, failure: result[:status].exitstatus, harness: benchmark_harness } + end + command = "#{RactorCounts::ENV_VAR}=#{count} #{result[:command]}" + blobs_by_count[count] = process_benchmark_result(result_json_path, command) + end + + data = RactorBreakdown.merge(blobs_by_count) + if caller_json_path + FileUtils.mkdir_p(File.dirname(caller_json_path)) + File.write(caller_json_path, JSON.pretty_generate(data)) + end + { name: entry.name, data: data, harness: benchmark_harness } + end + + # Clear project-level Bundler environment so benchmarks run in a clean context. + # Benchmarks that need Bundler (e.g., railsbench) set up their own via use_gemfile. + def run_benchmark_process(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: false) + if defined?(Bundler) + Bundler.with_unbundled_env do + run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet) + end + else + run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet) + end + end + def process_benchmark_result(result_json_path, command, delete_file: true) JSON.parse(File.read(result_json_path)).tap do |json| json["command_line"] = command diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index f3cdb613..3760b819 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -333,6 +333,69 @@ assert_empty bench_failures end + it 'runs each Ractor count in its own process and merges the results' do + File.write('benchmarks/per_count.rb', <<~RUBY) + require 'json' + count = ENV.fetch('RUBY_BENCH_RACTORS') + result = { + 'warmup' => [], + 'bench' => [count.to_f], + 'bench_by_ractors' => { count => [count.to_f] }, + 'rss' => (count.to_i + 1) * 1000, + 'pid' => Process.pid + } + File.write(ENV['RESULT_JSON_PATH'], JSON.generate(result)) + RUBY + File.write('benchmarks.yml', YAML.dump('per_count' => { 'category' => 'other', 'ractor' => true })) + + custom_json_path = File.join(@out_path, 'nested', 'custom_result.json') + ENV['RESULT_JSON_PATH'] = custom_json_path + ENV['RUBY_BENCH_RACTORS'] = '0,2,1' + suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness', no_pinning: true) + + bench_data, bench_failures = nil + capture_io do + bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') + end + + assert_empty bench_failures + data = bench_data['per_count'] + assert_equal({ '0' => [0.0], '1' => [1.0], '2' => [2.0] }, data['bench_by_ractors']) + pids = data['results_by_ractors'].values.map { |process| process['pid'] } + assert_equal 3, pids.uniq.size + refute data.key?('rss') + assert_equal 1000, data['results_by_ractors']['0']['rss'] + assert_match(/\ARUBY_BENCH_RACTORS=1 /, data['results_by_ractors']['1']['command_line']) + assert_equal data, JSON.parse(File.read(custom_json_path)) + assert_empty Dir.glob(File.join(@out_path, 'temp*.json')) + ensure + ENV.delete('RUBY_BENCH_RACTORS') + end + + it 'reports a Ractor benchmark as failed when one count fails' do + File.write('benchmarks/fails_at_two.rb', <<~RUBY) + require 'json' + count = ENV.fetch('RUBY_BENCH_RACTORS') + exit(3) if count == '2' + File.write(ENV['RESULT_JSON_PATH'], JSON.generate('warmup' => [], 'bench' => [1.0], 'rss' => 1)) + RUBY + File.write('benchmarks.yml', YAML.dump('fails_at_two' => { 'category' => 'other', 'ractor' => true })) + + ENV['RUBY_BENCH_RACTORS'] = '0,2,4' + suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness', no_pinning: true) + + bench_data, bench_failures = nil + capture_io do + bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') + end + + assert_empty bench_data + assert_equal 3, bench_failures['fails_at_two'] + assert_empty Dir.glob(File.join(@out_path, 'temp*.json')) + ensure + ENV.delete('RUBY_BENCH_RACTORS') + end + it 'expands pre_init when provided' do # Create a pre_init file pre_init_file = File.join(@temp_dir, 'pre_init.rb') From dbb73beccbac49baf091f6bab7da411ae0e7dcbf Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 14:07:43 +0100 Subject: [PATCH 07/10] Show per-count Ractor results in zjit_diff Each count now runs in its own process, so ZJIT stats and maxrss only mean something per count. zjit_diff now reports each count as "name (r=N)", and a benchmark name filter selects all of its counts. --- misc/zjit_diff.rb | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/misc/zjit_diff.rb b/misc/zjit_diff.rb index 4592f085..1b4acda5 100755 --- a/misc/zjit_diff.rb +++ b/misc/zjit_diff.rb @@ -8,6 +8,7 @@ # Pass --help to see options. require 'json' +require_relative '../lib/ractor_breakdown' class ZjitDiff DEFAULT_THRESHOLD_PCT = 5.0 # Percentage change to highlight @@ -83,7 +84,8 @@ class ZjitDiff def initialize(path, threshold_pct: DEFAULT_THRESHOLD_PCT, minimum_diff: DEFAULT_MINIMUM_DIFF, limit: DEFAULT_LIMIT, benchmarks: nil) @data = JSON.parse(File.read(path)) @metadata = @data['metadata'] - @raw_data = @data['raw_data'] + @base_names = {} + @raw_data = expand_ractor_counts(@data['raw_data']) @ruby_names = @raw_data.keys @threshold_pct = threshold_pct @minimum_diff = minimum_diff @@ -170,7 +172,29 @@ def print_header def benchmarks @benchmarks ||= begin all = @raw_data.values.first.keys - @benchmark_filter ? all & @benchmark_filter : all + if @benchmark_filter + all.select { |name| @benchmark_filter.include?(name) || @benchmark_filter.include?(@base_names[name]) } + else + all + end + end + end + + def expand_ractor_counts(raw_data) + raw_data.transform_values do |benchmarks| + benchmarks.each_with_object({}) do |(name, blob), expanded| + unless blob.is_a?(Hash) && blob['results_by_ractors'].is_a?(Hash) + expanded[name] = blob + next + end + + breakdown = blob['bench_by_ractors'] + breakdown.keys.map { |count| Integer(count) }.sort.each do |count| + label = "#{name} (r=#{count})" + @base_names[label] = name + expanded[label] = RactorBreakdown.per_count_blob(blob, breakdown, count) + end + end end end From 298fdaac0b02aa2d1108066f6253923f301dcd05 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 14:07:43 +0100 Subject: [PATCH 08/10] Document per-count Ractor processes --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index db115f02..4eedfa81 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,29 @@ intended to be used with any harness except `harness-ractor`. Note: The `harness-ractor` harness is automatically selected when using these categories, so there's no need to specify `--harness` manually. +### Ractor counts + +The Ractor harness measures each benchmark at 0 (the main Ractor only), 1, 2, +4, 6, and 8 Ractors. Set `RUBY_BENCH_RACTORS` to a comma-separated list to +change the counts, for example `RUBY_BENCH_RACTORS=0,2,8`. + +`run_benchmarks.rb` starts a fresh process for each count. Heap pages, the GC +page pool, and JIT state therefore cannot carry over from one count to the +next. Each process runs `WARMUP_ITRS` warmup iterations at its own count +before the measured iterations. + +The JSON output keeps one blob per benchmark. `bench_by_ractors` and +`gc_by_ractors` hold the measurements for each count. `results_by_ractors` +holds the process-level data of each count: `rss`, `maxrss`, YJIT or ZJIT +stats, and `command_line`. The blob has no top-level `rss`, `maxrss`, or JIT +stats, because no single process ran all counts. + +The text summary, the CSV output, and `misc/zjit_diff.rb` show one row for +each count, with the RSS and JIT stats of that count's process. + +When you run a benchmark directly with `-Iharness-ractor`, the harness runs +all counts in one process, one count after another. + ## Ruby options By default, ruby-bench benchmarks the Ruby used for `run_benchmarks.rb`. From 44d1b14e6b416ce2eb8d5cf98b67aa294257cccf Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 8 Oct 2026 13:13:35 +0100 Subject: [PATCH 09/10] Record Ractor warmup times for each count The Ractor harness ran WARMUP_ITRS iterations at each count but threw the times away and wrote an empty warmup array. The 1st itr column then fell back to bench[0], so for Ractor rows it compared two warm, measured iterations instead of the cold first iteration of each count's process. In this commit we keep the warmup times for each count in warmup_by_ractors, and RactorBreakdown gives each count row its own warmup. The 1st itr column now uses the same warmup[0] || bench[0] rule as the default harness. The harness also prints warmup iterations, as the default harness does. In --ractor-gc mode we keep only the wall time of warmup iterations, so gc_by_ractors still holds measured iterations only. --- README.md | 17 ++++++----- harness-ractor/harness.rb | 48 +++++++++++++++++------------- lib/ractor_breakdown.rb | 9 +++--- test/benchmark_runner_cli_test.rb | 3 ++ test/ractor_breakdown_test.rb | 26 ++++++++++++---- test/ractor_gc_harness_test.rb | 2 ++ test/ractor_harness_test.rb | 5 +++- test/results_table_builder_test.rb | 11 +++++-- 8 files changed, 81 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 4eedfa81..8e7cc976 100644 --- a/README.md +++ b/README.md @@ -174,13 +174,16 @@ change the counts, for example `RUBY_BENCH_RACTORS=0,2,8`. `run_benchmarks.rb` starts a fresh process for each count. Heap pages, the GC page pool, and JIT state therefore cannot carry over from one count to the next. Each process runs `WARMUP_ITRS` warmup iterations at its own count -before the measured iterations. - -The JSON output keeps one blob per benchmark. `bench_by_ractors` and -`gc_by_ractors` hold the measurements for each count. `results_by_ractors` -holds the process-level data of each count: `rss`, `maxrss`, YJIT or ZJIT -stats, and `command_line`. The blob has no top-level `rss`, `maxrss`, or JIT -stats, because no single process ran all counts. +before the measured iterations. As in the default harness, the harness prints +each warmup iteration and records its time. + +The JSON output keeps one blob per benchmark. `warmup_by_ractors`, +`bench_by_ractors`, and `gc_by_ractors` hold the measurements for each count. +`warmup_by_ractors` holds wall times only. The `--ractor-gc` mode does not +keep GC samples for warmup iterations. `results_by_ractors` holds the +process-level data of each count: `rss`, `maxrss`, YJIT or ZJIT stats, and +`command_line`. The blob has no top-level `rss`, `maxrss`, or JIT stats, +because no single process ran all counts. The text summary, the CSV output, and `misc/zjit_diff.rb` show one row for each count, with the RSS and JIT stats of that count's process. diff --git a/harness-ractor/harness.rb b/harness-ractor/harness.rb index c34975f1..9217d957 100644 --- a/harness-ractor/harness.rb +++ b/harness-ractor/harness.rb @@ -62,17 +62,18 @@ def check_ractor_gc_support end def run_benchmark_timing(warmup_itrs, bench_itrs, ractor_args, &block) - stats = Hash.new { |h,k| h[k] = [] } + warmups = {} + stats = {} RACTORS.each do |rs| - warmup_itrs.times { run_timing_iteration(rs, ractor_args, &block) } - bench_itrs.times do |i| + times = Array.new(warmup_itrs + bench_itrs) do |i| time = run_timing_iteration(rs, ractor_args, &block) - stats[rs] << time puts "%-3s %4s %6s" % ["#{rs}", "##{i + 1}:", "#{(1000 * time).to_i}ms"] + time end + warmups[rs], stats[rs] = times[0...warmup_itrs], times[warmup_itrs..] end - return_results([], stats.values.flatten, bench_by_ractors: stats) + return_results(warmups.values.flatten, stats.values.flatten, warmup_by_ractors: warmups, bench_by_ractors: stats) end def run_timing_iteration(rs, ractor_args, &block) @@ -103,7 +104,8 @@ def run_timing_iteration(rs, ractor_args, &block) }.freeze def run_benchmark_gc(warmup_itrs, bench_itrs, block, ractor_args, gc_config:) - stats = Hash.new { |h,k| h[k] = [] } + warmups = {} + stats = {} gc_by_ractors = {} header = +"r: itr: time gc_total marking sweeping gc_count major minor global" @@ -112,34 +114,37 @@ def run_benchmark_gc(warmup_itrs, bench_itrs, block, ractor_args, gc_config:) puts "(* controller-observed compacting cycles; may overlap global counts and is not additive.)" if CONTROLLER_GC_SERIES.any? RACTORS.each do |rs| + warmups[rs] = [] + stats[rs] = [] group = { "gc_worker_samples" => [] } group["gc_controller_samples"] = [] if rs > 0 series = Hash.new { |h,k| h[k] = [] } - warmup_itrs.times { run_ractor_gc_iteration(rs, ractor_args, &block) } - - num_itrs = 0 - while num_itrs < bench_itrs - num_itrs += 1 + (warmup_itrs + bench_itrs).times do |i| elapsed, worker_samples, controller_sample, controller_deltas = run_ractor_gc_iteration(rs, ractor_args, &block) - stats[rs] << elapsed - group["gc_worker_samples"] << worker_samples - group["gc_controller_samples"] << controller_sample if controller_sample - agg = GCStats.aggregate(worker_samples) total_ms = agg["gc_total_time_ns"]&.fdiv(1_000_000) - RACTOR_GC_SERIES.each do |series_name, field| - series[series_name] << (field == "gc_total_time_ns" ? total_ms : agg[field]) - end - CONTROLLER_GC_SERIES.each_key { |series_name| series[series_name] << controller_deltas[series_name] } - itr_str = "%-3s %4s %6s" % [rs, "##{num_itrs}:", "#{(1000 * elapsed).to_i}ms"] + itr_str = "%-3s %4s %6s" % [rs, "##{i + 1}:", "#{(1000 * elapsed).to_i}ms"] itr_str << " %8s" % (total_ms ? "%.1fms" % total_ms : "N/A") itr_str << " %8s" % (agg["gc_marking_time"] ? "#{agg["gc_marking_time"]}ms" : "N/A") itr_str << " %8s" % (agg["gc_sweeping_time"] ? "#{agg["gc_sweeping_time"]}ms" : "N/A") itr_str << " %9s %9s %9s %9s" % [agg["gc_count"], agg["gc_major_count"], agg["gc_minor_count"], agg["gc_global_count"]].map { |v| v.nil? ? "N/A" : v.to_s } CONTROLLER_GC_SERIES.each_key { |series_name| itr_str << " %9s" % (controller_deltas[series_name] || "N/A") } puts itr_str + + if i < warmup_itrs + warmups[rs] << elapsed + next + end + + stats[rs] << elapsed + group["gc_worker_samples"] << worker_samples + group["gc_controller_samples"] << controller_sample if controller_sample + RACTOR_GC_SERIES.each do |series_name, field| + series[series_name] << (field == "gc_total_time_ns" ? total_ms : agg[field]) + end + CONTROLLER_GC_SERIES.each_key { |series_name| series[series_name] << controller_deltas[series_name] } end series.each do |name, values| @@ -149,6 +154,7 @@ def run_benchmark_gc(warmup_itrs, bench_itrs, block, ractor_args, gc_config:) end extra = { + warmup_by_ractors: warmups, bench_by_ractors: stats, gc_scope: "ractor-local-workload", gc_stat_scope: "ractor-local", @@ -156,7 +162,7 @@ def run_benchmark_gc(warmup_itrs, bench_itrs, block, ractor_args, gc_config:) gc_by_ractors: gc_by_ractors, } extra[:gc_config] = gc_config if gc_config - return_results([], stats.values.flatten, **extra) + return_results(warmups.values.flatten, stats.values.flatten, **extra) end def controller_gc_snapshot diff --git a/lib/ractor_breakdown.rb b/lib/ractor_breakdown.rb index 1f318819..8d58d58b 100644 --- a/lib/ractor_breakdown.rb +++ b/lib/ractor_breakdown.rb @@ -2,7 +2,7 @@ module RactorBreakdown KEY_SEP = "\x00" - MEASUREMENT_KEYS = %w[warmup bench bench_by_ractors gc_by_ractors].freeze + MEASUREMENT_KEYS = %w[warmup bench warmup_by_ractors bench_by_ractors gc_by_ractors].freeze PROCESS_KEYS = %w[rss maxrss yjit_stats zjit_stats zjit_stats_string command_line].freeze Result = Struct.new(:bench_data, :groups) @@ -27,8 +27,9 @@ def merge(blobs_by_count) !PROCESS_KEYS.include?(k) && rest.all? { |data| data.key?(k) && data[k] == v } end - merged['warmup'] = [] + merged['warmup'] = counts.flat_map { |count| blobs_by_count[count]['warmup'] } merged['bench'] = counts.flat_map { |count| blobs_by_count[count]['bench'] } + merged['warmup_by_ractors'] = counts.to_h { |count| [count.to_s, blobs_by_count[count]['warmup']] } merged['bench_by_ractors'] = counts.to_h { |count| [count.to_s, blobs_by_count[count]['bench']] } gc_by_ractors = counts.filter_map do |count| group = blobs_by_count[count].dig('gc_by_ractors', count.to_s) @@ -66,11 +67,11 @@ def expand(bench_data) end def per_count_blob(blob, breakdown, count) - per_count = blob.reject { |k, _| k == 'bench_by_ractors' || k == 'gc_by_ractors' || k == 'results_by_ractors' || k == 'bench' } + per_count = blob.reject { |k, _| %w[warmup_by_ractors bench_by_ractors gc_by_ractors results_by_ractors bench].include?(k) } process_data = blob['results_by_ractors'] per_count.merge!(process_data[count.to_s]) if process_data.is_a?(Hash) && process_data.key?(count.to_s) per_count['bench'] = breakdown[count.to_s] - per_count['warmup'] = [] + per_count['warmup'] = blob.fetch('warmup_by_ractors').fetch(count.to_s) gc_by_ractors = blob['gc_by_ractors'] if gc_by_ractors.is_a?(Hash) && gc_by_ractors.key?(count.to_s) per_count.merge!(gc_by_ractors[count.to_s]) diff --git a/test/benchmark_runner_cli_test.rb b/test/benchmark_runner_cli_test.rb index 1908ac90..c3f24226 100644 --- a/test/benchmark_runner_cli_test.rb +++ b/test/benchmark_runner_cli_test.rb @@ -98,6 +98,7 @@ def create_args(overrides = {}) 'symbol-name-ractor' => { 'warmup' => [], 'bench' => [1.0, 2.0], + 'warmup_by_ractors' => { '0' => [], '2' => [] }, 'bench_by_ractors' => { '0' => [1.0], '2' => [2.0] }, 'rss' => 10 * 1024 * 1024 } @@ -144,6 +145,7 @@ def create_args(overrides = {}) 'bench' => [1.0, 2.0], 'rss' => 10 * 1024 * 1024, 'gc_scope' => 'ractor-local-workload', + 'warmup_by_ractors' => { '0' => [], '2' => [] }, 'bench_by_ractors' => { '0' => [1.0], '2' => [2.0] }, 'gc_by_ractors' => { '0' => gc_group.call(4.0, 1, 3), '2' => gc_group.call(12.0, 2, 6) } } @@ -183,6 +185,7 @@ def create_args(overrides = {}) 'bench' => [1.0, 2.0], 'rss' => 10 * 1024 * 1024, 'gc_scope' => 'ractor-local-workload', + 'warmup_by_ractors' => { '0' => [], '2' => [] }, 'bench_by_ractors' => { '0' => [1.0], '2' => [2.0] } } } diff --git a/test/ractor_breakdown_test.rb b/test/ractor_breakdown_test.rb index 7f130189..62eac6e4 100644 --- a/test/ractor_breakdown_test.rb +++ b/test/ractor_breakdown_test.rb @@ -21,12 +21,16 @@ 'ruby' => { 'symbol-name-ractor' => { 'bench' => [1.0, 2.0, 3.0, 4.0], + 'warmup_by_ractors' => { + '0' => [5.0], + '2' => [6.0] + }, 'bench_by_ractors' => { '0' => [1.0, 1.1], '2' => [2.0, 2.2] }, 'rss' => 555, - 'warmup' => [] + 'warmup' => [5.0, 6.0] } } } @@ -39,6 +43,9 @@ assert_equal [1.0, 1.1], exe[key0]['bench'] assert_equal [2.0, 2.2], exe[key2]['bench'] + assert_equal [5.0], exe[key0]['warmup'] + assert_equal [6.0], exe[key2]['warmup'] + refute exe[key0].key?('warmup_by_ractors') # process-wide fields are shared assert_equal 555, exe[key0]['rss'] assert_equal 555, exe[key2]['rss'] @@ -51,6 +58,7 @@ 'ruby' => { 'symbol-name-ractor' => { 'bench' => [], + 'warmup_by_ractors' => { '8' => [], '0' => [], '2' => [] }, 'bench_by_ractors' => { '8' => [1.0], '0' => [1.0], '2' => [1.0] } } } @@ -72,6 +80,7 @@ blob = lambda do { 'bench' => [], + 'warmup_by_ractors' => { '0' => [], '1' => [] }, 'bench_by_ractors' => { '0' => [1.0], '1' => [2.0] } } end @@ -91,6 +100,7 @@ it 'merges only the matching count\'s gc_by_ractors entry into each synthetic blob' do blob = { 'bench' => [3.0], + 'warmup_by_ractors' => { '0' => [], '2' => [] }, 'bench_by_ractors' => { '0' => [1.0], '2' => [2.0] }, 'gc_scope' => 'ractor-local-workload', 'gc_by_ractors' => { @@ -144,6 +154,7 @@ 'ruby' => { 'r' => { 'bench' => [1.0], + 'warmup_by_ractors' => { '0' => [] }, 'bench_by_ractors' => { '0' => [1.0] }, 'gc_scope' => 'ractor-local-workload', 'gc_stat_scope' => 'ractor-local', @@ -167,11 +178,12 @@ end describe '.merge' do - def child_blob(count, bench:, rss:, zjit_calls:) + def child_blob(count, warmup:, bench:, rss:, zjit_calls:) { 'RUBY_DESCRIPTION' => 'ruby 4.1.0', - 'warmup' => [], + 'warmup' => warmup, 'bench' => bench, + 'warmup_by_ractors' => { count.to_s => warmup }, 'bench_by_ractors' => { count.to_s => bench }, 'gc_scope' => 'ractor-local-workload', 'gc_by_ractors' => { count.to_s => { 'gc_count_bench' => [count * 10] } }, @@ -184,10 +196,12 @@ def child_blob(count, bench:, rss:, zjit_calls:) it 'keeps process-level data per count so each expanded row shows its own process' do merged = RactorBreakdown.merge( - 2 => child_blob(2, bench: [2.0, 2.1], rss: 300, zjit_calls: 7), - 0 => child_blob(0, bench: [1.0, 1.1], rss: 100, zjit_calls: 5) + 2 => child_blob(2, warmup: [9.0], bench: [2.0, 2.1], rss: 300, zjit_calls: 7), + 0 => child_blob(0, warmup: [8.0], bench: [1.0, 1.1], rss: 100, zjit_calls: 5) ) + assert_equal({ '0' => [8.0], '2' => [9.0] }, merged['warmup_by_ractors']) + assert_equal [8.0, 9.0], merged['warmup'] assert_equal({ '0' => [1.0, 1.1], '2' => [2.0, 2.1] }, merged['bench_by_ractors']) assert_equal [1.0, 1.1, 2.0, 2.1], merged['bench'] refute merged.key?('rss') @@ -203,6 +217,8 @@ def child_blob(count, bench:, rss:, zjit_calls:) r2 = exe["r\x002"] assert_equal [1.0, 1.1], r0['bench'] + assert_equal [8.0], r0['warmup'] + assert_equal [9.0], r2['warmup'] assert_equal 100, r0['rss'] assert_equal 300, r2['rss'] assert_equal({ 'calls' => 5 }, r0['zjit_stats']) diff --git a/test/ractor_gc_harness_test.rb b/test/ractor_gc_harness_test.rb index 7ad82e38..facb63a7 100644 --- a/test/ractor_gc_harness_test.rb +++ b/test/ractor_gc_harness_test.rb @@ -163,6 +163,8 @@ def run_workload(body) assert_equal 'ractor-local', data['gc_measure_total_time_scope'] assert_kind_of Hash, data['gc_config'] assert_equal %w[0 1 2], data['bench_by_ractors'].keys.sort + assert_equal({ '0' => 1, '1' => 1, '2' => 1 }, data['warmup_by_ractors'].transform_values(&:size)) + assert_equal data['warmup_by_ractors'].values_at('0', '1', '2').flatten, data['warmup'] assert_equal %w[0 1 2], data['gc_by_ractors'].keys.sort if data['gc_by_ractors'].values.any? { |group| group.key?('gc_controller_compact_count_bench') } diff --git a/test/ractor_harness_test.rb b/test/ractor_harness_test.rb index a1c12207..e68bc7de 100644 --- a/test/ractor_harness_test.rb +++ b/test/ractor_harness_test.rb @@ -30,7 +30,10 @@ stdout, stderr, status = Open3.capture3(env, RbConfig.ruby, "-I#{File.join(root, 'harness-ractor')}", script) assert status.success?, "workload failed:\n#{stdout}\n#{stderr}" - assert_equal ['2'], JSON.parse(File.read(result_path))['bench_by_ractors'].keys + data = JSON.parse(File.read(result_path)) + assert_equal({ '2' => 2 }, data['warmup_by_ractors'].transform_values(&:size)) + assert_equal({ '2' => 1 }, data['bench_by_ractors'].transform_values(&:size)) + assert_equal data['warmup_by_ractors']['2'], data['warmup'] assert_equal ['2'] * 6, File.readlines(calls_path, chomp: true), '(2 warmup + 1 measured) iterations x 2 Ractors' end end diff --git a/test/results_table_builder_test.rb b/test/results_table_builder_test.rb index 97246775..a98764d9 100644 --- a/test/results_table_builder_test.rb +++ b/test/results_table_builder_test.rb @@ -85,16 +85,18 @@ raw = { 'master' => { 'symbol-name-ractor' => { - 'warmup' => [], + 'warmup' => [3.0, 8.0], 'bench' => [1.0, 2.0], + 'warmup_by_ractors' => { '0' => [3.0], '2' => [8.0] }, 'bench_by_ractors' => { '0' => [1.0, 1.0], '2' => [2.0, 2.0] }, 'rss' => 10 * 1024 * 1024 } }, 'exp' => { 'symbol-name-ractor' => { - 'warmup' => [], + 'warmup' => [1.0, 2.0], 'bench' => [0.5, 1.0], + 'warmup_by_ractors' => { '0' => [1.0], '2' => [2.0] }, 'bench_by_ractors' => { '0' => [0.5, 0.5], '2' => [1.0, 1.0] }, 'rss' => 10 * 1024 * 1024 } @@ -119,6 +121,10 @@ assert_equal '', table[2][0] assert_equal '2', table[2][1] + # 1st itr uses each count's first warmup: r=0 3000ms vs 1000ms, r=2 8000ms vs 2000ms + assert_in_delta 3.0, table[1][4], 0.01 + assert_in_delta 4.0, table[2][4], 0.01 + # count=0 row: master 1000ms vs exp 500ms => ratio 2.0 assert_in_delta 2.0, table[1][5].to_f, 0.01 # count=2 row: master 2000ms vs exp 1000ms => ratio 2.0 @@ -860,6 +866,7 @@ def ractor_gc_blob(groups) 'bench' => groups.values.flat_map { |g| g[:bench] }, 'rss' => 10 * 1024 * 1024, 'gc_scope' => 'ractor-local-workload', + 'warmup_by_ractors' => groups.transform_values { [] }, 'bench_by_ractors' => groups.transform_values { |g| g[:bench] }, 'gc_by_ractors' => groups.transform_values { |g| g[:gc] } } From 41951092723139ff2329ae3a6bef120bbafe8aad Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 8 Oct 2026 13:13:46 +0100 Subject: [PATCH 10/10] Say "first iteration" in the 1st itr legend The legend said "first benchmarking iteration", which reads as the first measured iteration. The column uses warmup[0] when warmups ran. --- lib/benchmark_runner.rb | 2 +- test/benchmark_runner_test.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/benchmark_runner.rb b/lib/benchmark_runner.rb index 38f9222e..ffe77759 100644 --- a/lib/benchmark_runner.rb +++ b/lib/benchmark_runner.rb @@ -74,7 +74,7 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_ unless other_names.empty? output_str << "Legend:\n" other_names.each do |name| - output_str << "- #{name} 1st itr: ratio of #{base_name}/#{name} time for the first benchmarking iteration.\n" + output_str << "- #{name} 1st itr: ratio of #{base_name}/#{name} time for the first iteration.\n" output_str << "- #{base_name}/#{name}: ratio of #{base_name}/#{name} time. Higher is better for #{name}. Above 1 represents a speedup.\n" if include_rss output_str << "- RSS #{base_name}/#{name}: ratio of #{base_name}/#{name} RSS. Higher is better for #{name}. Above 1 means lower memory usage.\n" diff --git a/test/benchmark_runner_test.rb b/test/benchmark_runner_test.rb index 7792d8b5..b8c4ad75 100644 --- a/test/benchmark_runner_test.rb +++ b/test/benchmark_runner_test.rb @@ -385,7 +385,6 @@ assert_includes result, 'ruby-base: ruby 3.3.0' assert_includes result, 'ruby-yjit: ruby 3.3.0 +YJIT' assert_includes result, 'Legend:' - assert_includes result, '- ruby-yjit 1st itr: ratio of ruby-base/ruby-yjit time for the first benchmarking iteration.' assert_includes result, '- ruby-base/ruby-yjit: ratio of ruby-base/ruby-yjit time. Higher is better for ruby-yjit. Above 1 represents a speedup.' refute_includes result, "p < 0.001" end