diff --git a/README.md b/README.md index db115f02..8e7cc976 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,32 @@ intended to be used with any harness except `harness-ractor`. Note: The `harness-ractor` harness is automatically selected when using these categories, so there's no need to specify `--harness` manually. +### Ractor counts + +The Ractor harness measures each benchmark at 0 (the main Ractor only), 1, 2, +4, 6, and 8 Ractors. Set `RUBY_BENCH_RACTORS` to a comma-separated list to +change the counts, for example `RUBY_BENCH_RACTORS=0,2,8`. + +`run_benchmarks.rb` starts a fresh process for each count. Heap pages, the GC +page pool, and JIT state therefore cannot carry over from one count to the +next. Each process runs `WARMUP_ITRS` warmup iterations at its own count +before the measured iterations. As in the default harness, the harness prints +each warmup iteration and records its time. + +The JSON output keeps one blob per benchmark. `warmup_by_ractors`, +`bench_by_ractors`, and `gc_by_ractors` hold the measurements for each count. +`warmup_by_ractors` holds wall times only. The `--ractor-gc` mode does not +keep GC samples for warmup iterations. `results_by_ractors` holds the +process-level data of each count: `rss`, `maxrss`, YJIT or ZJIT stats, and +`command_line`. The blob has no top-level `rss`, `maxrss`, or JIT stats, +because no single process ran all counts. + +The text summary, the CSV output, and `misc/zjit_diff.rb` show one row for +each count, with the RSS and JIT stats of that count's process. + +When you run a benchmark directly with `-Iharness-ractor`, the harness runs +all counts in one process, one count after another. + ## Ruby options By default, ruby-bench benchmarks the Ruby used for `run_benchmarks.rb`. diff --git a/harness-ractor/harness.rb b/harness-ractor/harness.rb index f7c42667..9217d957 100644 --- a/harness-ractor/harness.rb +++ b/harness-ractor/harness.rb @@ -17,18 +17,8 @@ {}.freeze end -default_ractors = [ - 0, # without ractor - 1, 2, 4, 6, 8#, 12, 16, 32 -] -if rs = ENV["RUBY_BENCH_RACTORS"] - rs = rs.split(",").map(&:to_i) # If you want to include 0, you have to specify - rs = rs.sort.uniq - if rs.any? - ractors = rs - end -end -RACTORS = (ractors || default_ractors).freeze +require_relative '../lib/ractor_counts' +RACTORS = RactorCounts.from_env unless Ractor.method_defined?(:join) class Ractor @@ -51,13 +41,11 @@ def run_benchmark(num_itrs_hint, ractor_args: [], &block) check_ractor_gc_support gc_config = GC.config.transform_keys(&:to_s) if GC.respond_to?(:config) GCStats.with_measure_total_time do - run_warmup(warmup_itrs, ractor_args, &block) - run_benchmark_gc(bench_itrs, Ractor.make_shareable(block), ractor_args, gc_config: gc_config) + run_benchmark_gc(warmup_itrs, bench_itrs, Ractor.make_shareable(block), ractor_args, gc_config: gc_config) end else puts "r: itr: time" - run_warmup(warmup_itrs, ractor_args, &block) - run_benchmark_timing(bench_itrs, ractor_args, &block) + run_benchmark_timing(warmup_itrs, bench_itrs, ractor_args, &block) end end @@ -73,41 +61,36 @@ def check_ractor_gc_support end end -def run_warmup(warmup_itrs, ractor_args, &block) - warmup_itrs.times do - args = ractor_args.empty? ? [] : ractor_deep_dup(ractor_args) - block.call(*([0] + args)) +def run_benchmark_timing(warmup_itrs, bench_itrs, ractor_args, &block) + warmups = {} + stats = {} + + RACTORS.each do |rs| + times = Array.new(warmup_itrs + bench_itrs) do |i| + time = run_timing_iteration(rs, ractor_args, &block) + puts "%-3s %4s %6s" % ["#{rs}", "##{i + 1}:", "#{(1000 * time).to_i}ms"] + time + end + warmups[rs], stats[rs] = times[0...warmup_itrs], times[warmup_itrs..] end + return_results(warmups.values.flatten, stats.values.flatten, warmup_by_ractors: warmups, bench_by_ractors: stats) end -def run_benchmark_timing(bench_itrs, ractor_args, &block) - stats = Hash.new { |h,k| h[k] = [] } - - RACTORS.each do |rs| - num_itrs = 0 - while num_itrs < bench_itrs - before = Process.clock_gettime(Process::CLOCK_MONOTONIC) - if rs.zero? - block.call *([rs] + ractor_deep_dup(ractor_args)) - else - rs_list = [] - rs.times do - rs_list << Ractor.new(*([rs] + ractor_args), &block) # ractor_args are copied - end - while rs_list.any? - r, _obj = Ractor.select(*rs_list) - rs_list.delete(r) - end - end - num_itrs += 1 - time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - before - time_ms = (1000 * time).to_i - itr_str = "%-3s %4s %6s" % ["#{rs}", "##{num_itrs}:", "#{time_ms}ms"] - stats[rs] << time - puts itr_str +def run_timing_iteration(rs, ractor_args, &block) + before = Process.clock_gettime(Process::CLOCK_MONOTONIC) + if rs.zero? + block.call *([rs] + ractor_deep_dup(ractor_args)) + else + rs_list = [] + rs.times do + rs_list << Ractor.new(*([rs] + ractor_args), &block) # ractor_args are copied + end + while rs_list.any? + r, _obj = Ractor.select(*rs_list) + rs_list.delete(r) end end - return_results([], stats.values.flatten, bench_by_ractors: stats) + Process.clock_gettime(Process::CLOCK_MONOTONIC) - before end RACTOR_GC_SERIES = { @@ -120,8 +103,9 @@ def run_benchmark_timing(bench_itrs, ractor_args, &block) "gc_total_time_bench" => "gc_total_time_ns", }.freeze -def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) - stats = Hash.new { |h,k| h[k] = [] } +def run_benchmark_gc(warmup_itrs, bench_itrs, block, ractor_args, gc_config:) + warmups = {} + stats = {} gc_by_ractors = {} header = +"r: itr: time gc_total marking sweeping gc_count major minor global" @@ -130,32 +114,37 @@ def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) puts "(* controller-observed compacting cycles; may overlap global counts and is not additive.)" if CONTROLLER_GC_SERIES.any? RACTORS.each do |rs| + warmups[rs] = [] + stats[rs] = [] group = { "gc_worker_samples" => [] } group["gc_controller_samples"] = [] if rs > 0 series = Hash.new { |h,k| h[k] = [] } - num_itrs = 0 - while num_itrs < bench_itrs - num_itrs += 1 + (warmup_itrs + bench_itrs).times do |i| elapsed, worker_samples, controller_sample, controller_deltas = run_ractor_gc_iteration(rs, ractor_args, &block) - stats[rs] << elapsed - group["gc_worker_samples"] << worker_samples - group["gc_controller_samples"] << controller_sample if controller_sample - agg = GCStats.aggregate(worker_samples) total_ms = agg["gc_total_time_ns"]&.fdiv(1_000_000) - RACTOR_GC_SERIES.each do |series_name, field| - series[series_name] << (field == "gc_total_time_ns" ? total_ms : agg[field]) - end - CONTROLLER_GC_SERIES.each_key { |series_name| series[series_name] << controller_deltas[series_name] } - itr_str = "%-3s %4s %6s" % [rs, "##{num_itrs}:", "#{(1000 * elapsed).to_i}ms"] + itr_str = "%-3s %4s %6s" % [rs, "##{i + 1}:", "#{(1000 * elapsed).to_i}ms"] itr_str << " %8s" % (total_ms ? "%.1fms" % total_ms : "N/A") itr_str << " %8s" % (agg["gc_marking_time"] ? "#{agg["gc_marking_time"]}ms" : "N/A") itr_str << " %8s" % (agg["gc_sweeping_time"] ? "#{agg["gc_sweeping_time"]}ms" : "N/A") itr_str << " %9s %9s %9s %9s" % [agg["gc_count"], agg["gc_major_count"], agg["gc_minor_count"], agg["gc_global_count"]].map { |v| v.nil? ? "N/A" : v.to_s } CONTROLLER_GC_SERIES.each_key { |series_name| itr_str << " %9s" % (controller_deltas[series_name] || "N/A") } puts itr_str + + if i < warmup_itrs + warmups[rs] << elapsed + next + end + + stats[rs] << elapsed + group["gc_worker_samples"] << worker_samples + group["gc_controller_samples"] << controller_sample if controller_sample + RACTOR_GC_SERIES.each do |series_name, field| + series[series_name] << (field == "gc_total_time_ns" ? total_ms : agg[field]) + end + CONTROLLER_GC_SERIES.each_key { |series_name| series[series_name] << controller_deltas[series_name] } end series.each do |name, values| @@ -165,6 +154,7 @@ def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) end extra = { + warmup_by_ractors: warmups, bench_by_ractors: stats, gc_scope: "ractor-local-workload", gc_stat_scope: "ractor-local", @@ -172,7 +162,7 @@ def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) gc_by_ractors: gc_by_ractors, } extra[:gc_config] = gc_config if gc_config - return_results([], stats.values.flatten, **extra) + return_results(warmups.values.flatten, stats.values.flatten, **extra) end def controller_gc_snapshot diff --git a/lib/benchmark_runner.rb b/lib/benchmark_runner.rb index 38f9222e..ffe77759 100644 --- a/lib/benchmark_runner.rb +++ b/lib/benchmark_runner.rb @@ -74,7 +74,7 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_ unless other_names.empty? output_str << "Legend:\n" other_names.each do |name| - output_str << "- #{name} 1st itr: ratio of #{base_name}/#{name} time for the first benchmarking iteration.\n" + output_str << "- #{name} 1st itr: ratio of #{base_name}/#{name} time for the first iteration.\n" output_str << "- #{base_name}/#{name}: ratio of #{base_name}/#{name} time. Higher is better for #{name}. Above 1 represents a speedup.\n" if include_rss output_str << "- RSS #{base_name}/#{name}: ratio of #{base_name}/#{name} RSS. Higher is better for #{name}. Above 1 means lower memory usage.\n" diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index ed59d2d1..da1a73c2 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -110,12 +110,14 @@ def run puts # Build the results table + csv_data, csv_layout = csv_view(bench_data) builder = ResultsTableBuilder.new( executable_names: ruby_descriptions.keys, - bench_data: bench_data, + bench_data: csv_data, include_rss: args.rss, include_pvalue: args.pvalue, - zjit_stats: args.zjit_stats + zjit_stats: args.zjit_stats, + row_layout: csv_layout ) table, format, gc_table, gc_format = builder.build @@ -158,6 +160,13 @@ def run private + def csv_view(bench_data) + breakdown = RactorBreakdown.expand(bench_data) + return [bench_data, FlatRowLayout.new] if breakdown.groups.empty? + + [breakdown.bench_data, RactorRowLayout.new(groups: breakdown.groups)] + end + def build_output_sections(executable_names, bench_data, bench_harnesses, bench_failures) ordered_names = sorted_benchmark_names(executable_names, bench_data) failed_names = bench_failures.values.flat_map(&:keys).uniq diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 4c22a12a..c387e770 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -10,6 +10,8 @@ require_relative 'benchmark_filter' require_relative 'benchmark_runner' require_relative 'benchmark_discovery' +require_relative 'ractor_breakdown' +require_relative 'ractor_counts' # BenchmarkSuite runs a collection of benchmarks and collects their results class BenchmarkSuite @@ -48,22 +50,16 @@ def run_benchmark(entry, ruby:, ruby_description:) env = benchmark_env(ruby) caller_json_path = ENV["RESULT_JSON_PATH"] quiet = ENV['BENCHMARK_QUIET'] == '1' - - result_json_path = caller_json_path || File.join(out_path, "temp#{Process.pid}.json") cmd_prefix = base_cmd(ruby_description, entry.name) - - # Clear project-level Bundler environment so benchmarks run in a clean context. - # Benchmarks that need Bundler (e.g., railsbench) set up their own via use_gemfile. benchmark_harness = benchmark_harness_for(entry.name) - result = if defined?(Bundler) - Bundler.with_unbundled_env do - run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet) - end - else - run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet) + if benchmark_harness == RACTOR_HARNESS + return run_ractor_benchmark(entry, ruby, cmd_prefix, env, benchmark_harness, caller_json_path, quiet: quiet) end + result_json_path = caller_json_path || File.join(out_path, "temp#{Process.pid}.json") + result = run_benchmark_process(entry.script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet) + if result[:success] { name: entry.name, data: process_benchmark_result(result_json_path, result[:command], delete_file: !caller_json_path), harness: benchmark_harness } else @@ -106,6 +102,42 @@ def setup_benchmark_directories end end + def run_ractor_benchmark(entry, ruby, cmd_prefix, env, benchmark_harness, caller_json_path, quiet: false) + blobs_by_count = {} + + RactorCounts.from_env.each do |count| + result_json_path = File.join(out_path, "temp#{Process.pid}_r#{count}.json") + count_env = env.merge(RactorCounts::ENV_VAR => count.to_s) + result = run_benchmark_process(entry.script_path, result_json_path, ruby, cmd_prefix, count_env, benchmark_harness, quiet: quiet) + + unless result[:success] + FileUtils.rm_f(result_json_path) + return { name: entry.name, failure: result[:status].exitstatus, harness: benchmark_harness } + end + command = "#{RactorCounts::ENV_VAR}=#{count} #{result[:command]}" + blobs_by_count[count] = process_benchmark_result(result_json_path, command) + end + + data = RactorBreakdown.merge(blobs_by_count) + if caller_json_path + FileUtils.mkdir_p(File.dirname(caller_json_path)) + File.write(caller_json_path, JSON.pretty_generate(data)) + end + { name: entry.name, data: data, harness: benchmark_harness } + end + + # Clear project-level Bundler environment so benchmarks run in a clean context. + # Benchmarks that need Bundler (e.g., railsbench) set up their own via use_gemfile. + def run_benchmark_process(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: false) + if defined?(Bundler) + Bundler.with_unbundled_env do + run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet) + end + else + run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet) + end + end + def process_benchmark_result(result_json_path, command, delete_file: true) JSON.parse(File.read(result_json_path)).tap do |json| json["command_line"] = command diff --git a/lib/ractor_breakdown.rb b/lib/ractor_breakdown.rb index 9a849595..8d58d58b 100644 --- a/lib/ractor_breakdown.rb +++ b/lib/ractor_breakdown.rb @@ -2,6 +2,8 @@ module RactorBreakdown KEY_SEP = "\x00" + MEASUREMENT_KEYS = %w[warmup bench warmup_by_ractors bench_by_ractors gc_by_ractors].freeze + PROCESS_KEYS = %w[rss maxrss yjit_stats zjit_stats zjit_stats_string command_line].freeze Result = Struct.new(:bench_data, :groups) @@ -15,6 +17,29 @@ def base_name(data_key) data_key.split(KEY_SEP, 2).first end + def merge(blobs_by_count) + counts = blobs_by_count.keys.sort + process_data = counts.to_h do |count| + [count.to_s, blobs_by_count[count].reject { |k, _| MEASUREMENT_KEYS.include?(k) }] + end + first, *rest = process_data.values + merged = first.select do |k, v| + !PROCESS_KEYS.include?(k) && rest.all? { |data| data.key?(k) && data[k] == v } + end + + merged['warmup'] = counts.flat_map { |count| blobs_by_count[count]['warmup'] } + merged['bench'] = counts.flat_map { |count| blobs_by_count[count]['bench'] } + merged['warmup_by_ractors'] = counts.to_h { |count| [count.to_s, blobs_by_count[count]['warmup']] } + merged['bench_by_ractors'] = counts.to_h { |count| [count.to_s, blobs_by_count[count]['bench']] } + gc_by_ractors = counts.filter_map do |count| + group = blobs_by_count[count].dig('gc_by_ractors', count.to_s) + [count.to_s, group] if group + end.to_h + merged['gc_by_ractors'] = gc_by_ractors unless gc_by_ractors.empty? + merged['results_by_ractors'] = process_data.transform_values { |data| data.reject { |k, _| merged.key?(k) } } + merged + end + def expand(bench_data) groups = {} new_data = {} @@ -42,9 +67,11 @@ def expand(bench_data) end def per_count_blob(blob, breakdown, count) - per_count = blob.reject { |k, _| k == 'bench_by_ractors' || k == 'gc_by_ractors' || k == 'bench' } + per_count = blob.reject { |k, _| %w[warmup_by_ractors bench_by_ractors gc_by_ractors results_by_ractors bench].include?(k) } + process_data = blob['results_by_ractors'] + per_count.merge!(process_data[count.to_s]) if process_data.is_a?(Hash) && process_data.key?(count.to_s) per_count['bench'] = breakdown[count.to_s] - per_count['warmup'] = [] + per_count['warmup'] = blob.fetch('warmup_by_ractors').fetch(count.to_s) gc_by_ractors = blob['gc_by_ractors'] if gc_by_ractors.is_a?(Hash) && gc_by_ractors.key?(count.to_s) per_count.merge!(gc_by_ractors[count.to_s]) diff --git a/lib/ractor_counts.rb b/lib/ractor_counts.rb new file mode 100644 index 00000000..15aada1b --- /dev/null +++ b/lib/ractor_counts.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module RactorCounts + DEFAULT = [0, 1, 2, 4, 6, 8].freeze + ENV_VAR = "RUBY_BENCH_RACTORS" + + module_function + + def from_env(env = ENV) + counts = env.fetch(ENV_VAR, "").split(",").map(&:to_i).sort.uniq + counts.empty? ? DEFAULT : counts.freeze + end +end diff --git a/lib/row_layout.rb b/lib/row_layout.rb index 92b47e5d..159fce63 100644 --- a/lib/row_layout.rb +++ b/lib/row_layout.rb @@ -32,7 +32,7 @@ def entries(bench_names) seen[base_name] = true members = @groups_by_base_name[base_name] - next [] unless members + next [Entry.new(data_key: data_key, label_cells: [data_key, ''])] unless members members.each_with_index.map do |(member_key, count), i| name_cell = i.zero? ? base_name : '' diff --git a/misc/zjit_diff.rb b/misc/zjit_diff.rb index 4592f085..1b4acda5 100755 --- a/misc/zjit_diff.rb +++ b/misc/zjit_diff.rb @@ -8,6 +8,7 @@ # Pass --help to see options. require 'json' +require_relative '../lib/ractor_breakdown' class ZjitDiff DEFAULT_THRESHOLD_PCT = 5.0 # Percentage change to highlight @@ -83,7 +84,8 @@ class ZjitDiff def initialize(path, threshold_pct: DEFAULT_THRESHOLD_PCT, minimum_diff: DEFAULT_MINIMUM_DIFF, limit: DEFAULT_LIMIT, benchmarks: nil) @data = JSON.parse(File.read(path)) @metadata = @data['metadata'] - @raw_data = @data['raw_data'] + @base_names = {} + @raw_data = expand_ractor_counts(@data['raw_data']) @ruby_names = @raw_data.keys @threshold_pct = threshold_pct @minimum_diff = minimum_diff @@ -170,7 +172,29 @@ def print_header def benchmarks @benchmarks ||= begin all = @raw_data.values.first.keys - @benchmark_filter ? all & @benchmark_filter : all + if @benchmark_filter + all.select { |name| @benchmark_filter.include?(name) || @benchmark_filter.include?(@base_names[name]) } + else + all + end + end + end + + def expand_ractor_counts(raw_data) + raw_data.transform_values do |benchmarks| + benchmarks.each_with_object({}) do |(name, blob), expanded| + unless blob.is_a?(Hash) && blob['results_by_ractors'].is_a?(Hash) + expanded[name] = blob + next + end + + breakdown = blob['bench_by_ractors'] + breakdown.keys.map { |count| Integer(count) }.sort.each do |count| + label = "#{name} (r=#{count})" + @base_names[label] = name + expanded[label] = RactorBreakdown.per_count_blob(blob, breakdown, count) + end + end end end diff --git a/test/benchmark_runner_cli_test.rb b/test/benchmark_runner_cli_test.rb index 7baf82f3..c3f24226 100644 --- a/test/benchmark_runner_cli_test.rb +++ b/test/benchmark_runner_cli_test.rb @@ -98,6 +98,7 @@ def create_args(overrides = {}) 'symbol-name-ractor' => { 'warmup' => [], 'bench' => [1.0, 2.0], + 'warmup_by_ractors' => { '0' => [], '2' => [] }, 'bench_by_ractors' => { '0' => [1.0], '2' => [2.0] }, 'rss' => 10 * 1024 * 1024 } @@ -144,6 +145,7 @@ def create_args(overrides = {}) 'bench' => [1.0, 2.0], 'rss' => 10 * 1024 * 1024, 'gc_scope' => 'ractor-local-workload', + 'warmup_by_ractors' => { '0' => [], '2' => [] }, 'bench_by_ractors' => { '0' => [1.0], '2' => [2.0] }, 'gc_by_ractors' => { '0' => gc_group.call(4.0, 1, 3), '2' => gc_group.call(12.0, 2, 6) } } @@ -183,6 +185,7 @@ def create_args(overrides = {}) 'bench' => [1.0, 2.0], 'rss' => 10 * 1024 * 1024, 'gc_scope' => 'ractor-local-workload', + 'warmup_by_ractors' => { '0' => [], '2' => [] }, 'bench_by_ractors' => { '0' => [1.0], '2' => [2.0] } } } @@ -197,6 +200,25 @@ def create_args(overrides = {}) end end + describe '#csv_view' do + it 'gives each Ractor count its own CSV row with that process RSS' do + cli = BenchmarkRunner::CLI.new(create_args) + mib = 1024 * 1024 + merged = RactorBreakdown.merge( + 0 => { 'warmup' => [], 'bench' => [1.0], 'rss' => 10 * mib }, + 2 => { 'warmup' => [], 'bench' => [2.0], 'rss' => 30 * mib } + ) + bench_data = { 'ruby' => { 'fib' => { 'warmup' => [], 'bench' => [0.1], 'rss' => 5 * mib }, 'gvl' => merged } } + + data, layout = cli.send(:csv_view, bench_data) + table, = ResultsTableBuilder.new(executable_names: ['ruby'], bench_data: data, include_rss: true, row_layout: layout).build + + assert_equal ['bench', 'ractors', 'ruby (ms)', 'RSS (MiB)'], table[0] + rows = table.drop(1).to_h { |row| [row[0..1], row[3]] } + assert_equal({ ['fib', ''] => 5.0, ['gvl', '0'] => 10.0, ['', '2'] => 30.0 }, rows) + end + end + describe '#run integration test' do it 'runs a simple benchmark end-to-end and produces all output files' do Dir.mktmpdir do |tmpdir| diff --git a/test/benchmark_runner_test.rb b/test/benchmark_runner_test.rb index 7792d8b5..b8c4ad75 100644 --- a/test/benchmark_runner_test.rb +++ b/test/benchmark_runner_test.rb @@ -385,7 +385,6 @@ assert_includes result, 'ruby-base: ruby 3.3.0' assert_includes result, 'ruby-yjit: ruby 3.3.0 +YJIT' assert_includes result, 'Legend:' - assert_includes result, '- ruby-yjit 1st itr: ratio of ruby-base/ruby-yjit time for the first benchmarking iteration.' assert_includes result, '- ruby-base/ruby-yjit: ratio of ruby-base/ruby-yjit time. Higher is better for ruby-yjit. Above 1 represents a speedup.' refute_includes result, "p < 0.001" end diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index f3cdb613..3760b819 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -333,6 +333,69 @@ assert_empty bench_failures end + it 'runs each Ractor count in its own process and merges the results' do + File.write('benchmarks/per_count.rb', <<~RUBY) + require 'json' + count = ENV.fetch('RUBY_BENCH_RACTORS') + result = { + 'warmup' => [], + 'bench' => [count.to_f], + 'bench_by_ractors' => { count => [count.to_f] }, + 'rss' => (count.to_i + 1) * 1000, + 'pid' => Process.pid + } + File.write(ENV['RESULT_JSON_PATH'], JSON.generate(result)) + RUBY + File.write('benchmarks.yml', YAML.dump('per_count' => { 'category' => 'other', 'ractor' => true })) + + custom_json_path = File.join(@out_path, 'nested', 'custom_result.json') + ENV['RESULT_JSON_PATH'] = custom_json_path + ENV['RUBY_BENCH_RACTORS'] = '0,2,1' + suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness', no_pinning: true) + + bench_data, bench_failures = nil + capture_io do + bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') + end + + assert_empty bench_failures + data = bench_data['per_count'] + assert_equal({ '0' => [0.0], '1' => [1.0], '2' => [2.0] }, data['bench_by_ractors']) + pids = data['results_by_ractors'].values.map { |process| process['pid'] } + assert_equal 3, pids.uniq.size + refute data.key?('rss') + assert_equal 1000, data['results_by_ractors']['0']['rss'] + assert_match(/\ARUBY_BENCH_RACTORS=1 /, data['results_by_ractors']['1']['command_line']) + assert_equal data, JSON.parse(File.read(custom_json_path)) + assert_empty Dir.glob(File.join(@out_path, 'temp*.json')) + ensure + ENV.delete('RUBY_BENCH_RACTORS') + end + + it 'reports a Ractor benchmark as failed when one count fails' do + File.write('benchmarks/fails_at_two.rb', <<~RUBY) + require 'json' + count = ENV.fetch('RUBY_BENCH_RACTORS') + exit(3) if count == '2' + File.write(ENV['RESULT_JSON_PATH'], JSON.generate('warmup' => [], 'bench' => [1.0], 'rss' => 1)) + RUBY + File.write('benchmarks.yml', YAML.dump('fails_at_two' => { 'category' => 'other', 'ractor' => true })) + + ENV['RUBY_BENCH_RACTORS'] = '0,2,4' + suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness', no_pinning: true) + + bench_data, bench_failures = nil + capture_io do + bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') + end + + assert_empty bench_data + assert_equal 3, bench_failures['fails_at_two'] + assert_empty Dir.glob(File.join(@out_path, 'temp*.json')) + ensure + ENV.delete('RUBY_BENCH_RACTORS') + end + it 'expands pre_init when provided' do # Create a pre_init file pre_init_file = File.join(@temp_dir, 'pre_init.rb') diff --git a/test/ractor_breakdown_test.rb b/test/ractor_breakdown_test.rb index 11e6335d..62eac6e4 100644 --- a/test/ractor_breakdown_test.rb +++ b/test/ractor_breakdown_test.rb @@ -21,12 +21,16 @@ 'ruby' => { 'symbol-name-ractor' => { 'bench' => [1.0, 2.0, 3.0, 4.0], + 'warmup_by_ractors' => { + '0' => [5.0], + '2' => [6.0] + }, 'bench_by_ractors' => { '0' => [1.0, 1.1], '2' => [2.0, 2.2] }, 'rss' => 555, - 'warmup' => [] + 'warmup' => [5.0, 6.0] } } } @@ -39,6 +43,9 @@ assert_equal [1.0, 1.1], exe[key0]['bench'] assert_equal [2.0, 2.2], exe[key2]['bench'] + assert_equal [5.0], exe[key0]['warmup'] + assert_equal [6.0], exe[key2]['warmup'] + refute exe[key0].key?('warmup_by_ractors') # process-wide fields are shared assert_equal 555, exe[key0]['rss'] assert_equal 555, exe[key2]['rss'] @@ -51,6 +58,7 @@ 'ruby' => { 'symbol-name-ractor' => { 'bench' => [], + 'warmup_by_ractors' => { '8' => [], '0' => [], '2' => [] }, 'bench_by_ractors' => { '8' => [1.0], '0' => [1.0], '2' => [1.0] } } } @@ -72,6 +80,7 @@ blob = lambda do { 'bench' => [], + 'warmup_by_ractors' => { '0' => [], '1' => [] }, 'bench_by_ractors' => { '0' => [1.0], '1' => [2.0] } } end @@ -91,6 +100,7 @@ it 'merges only the matching count\'s gc_by_ractors entry into each synthetic blob' do blob = { 'bench' => [3.0], + 'warmup_by_ractors' => { '0' => [], '2' => [] }, 'bench_by_ractors' => { '0' => [1.0], '2' => [2.0] }, 'gc_scope' => 'ractor-local-workload', 'gc_by_ractors' => { @@ -144,6 +154,7 @@ 'ruby' => { 'r' => { 'bench' => [1.0], + 'warmup_by_ractors' => { '0' => [] }, 'bench_by_ractors' => { '0' => [1.0] }, 'gc_scope' => 'ractor-local-workload', 'gc_stat_scope' => 'ractor-local', @@ -165,4 +176,57 @@ assert_equal({ 'implementation' => 'default' }, per_count['gc_config']) end end + + describe '.merge' do + def child_blob(count, warmup:, bench:, rss:, zjit_calls:) + { + 'RUBY_DESCRIPTION' => 'ruby 4.1.0', + 'warmup' => warmup, + 'bench' => bench, + 'warmup_by_ractors' => { count.to_s => warmup }, + 'bench_by_ractors' => { count.to_s => bench }, + 'gc_scope' => 'ractor-local-workload', + 'gc_by_ractors' => { count.to_s => { 'gc_count_bench' => [count * 10] } }, + 'rss' => rss, + 'maxrss' => 500, + 'zjit_stats' => { 'calls' => zjit_calls }, + 'command_line' => "RUBY_BENCH_RACTORS=#{count} ruby bench.rb" + } + end + + it 'keeps process-level data per count so each expanded row shows its own process' do + merged = RactorBreakdown.merge( + 2 => child_blob(2, warmup: [9.0], bench: [2.0, 2.1], rss: 300, zjit_calls: 7), + 0 => child_blob(0, warmup: [8.0], bench: [1.0, 1.1], rss: 100, zjit_calls: 5) + ) + + assert_equal({ '0' => [8.0], '2' => [9.0] }, merged['warmup_by_ractors']) + assert_equal [8.0, 9.0], merged['warmup'] + assert_equal({ '0' => [1.0, 1.1], '2' => [2.0, 2.1] }, merged['bench_by_ractors']) + assert_equal [1.0, 1.1, 2.0, 2.1], merged['bench'] + refute merged.key?('rss') + refute merged.key?('maxrss'), 'equal process-level values must stay per count' + assert_equal 'ruby 4.1.0', merged['RUBY_DESCRIPTION'] + assert_equal 'ractor-local-workload', merged['gc_scope'] + refute merged.key?('zjit_stats') + refute merged.key?('command_line') + assert_equal %w[rss maxrss zjit_stats command_line], merged['results_by_ractors']['0'].keys + + exe = RactorBreakdown.expand({ 'ruby' => { 'r' => merged } }).bench_data['ruby'] + r0 = exe["r\x000"] + r2 = exe["r\x002"] + + assert_equal [1.0, 1.1], r0['bench'] + assert_equal [8.0], r0['warmup'] + assert_equal [9.0], r2['warmup'] + assert_equal 100, r0['rss'] + assert_equal 300, r2['rss'] + assert_equal({ 'calls' => 5 }, r0['zjit_stats']) + assert_equal({ 'calls' => 7 }, r2['zjit_stats']) + assert_equal [0], r0['gc_count_bench'] + assert_equal [20], r2['gc_count_bench'] + assert_equal 'RUBY_BENCH_RACTORS=2 ruby bench.rb', r2['command_line'] + refute r0.key?('results_by_ractors') + end + end end diff --git a/test/ractor_gc_harness_test.rb b/test/ractor_gc_harness_test.rb index 7ad82e38..facb63a7 100644 --- a/test/ractor_gc_harness_test.rb +++ b/test/ractor_gc_harness_test.rb @@ -163,6 +163,8 @@ def run_workload(body) assert_equal 'ractor-local', data['gc_measure_total_time_scope'] assert_kind_of Hash, data['gc_config'] assert_equal %w[0 1 2], data['bench_by_ractors'].keys.sort + assert_equal({ '0' => 1, '1' => 1, '2' => 1 }, data['warmup_by_ractors'].transform_values(&:size)) + assert_equal data['warmup_by_ractors'].values_at('0', '1', '2').flatten, data['warmup'] assert_equal %w[0 1 2], data['gc_by_ractors'].keys.sort if data['gc_by_ractors'].values.any? { |group| group.key?('gc_controller_compact_count_bench') } diff --git a/test/ractor_harness_test.rb b/test/ractor_harness_test.rb new file mode 100644 index 00000000..e68bc7de --- /dev/null +++ b/test/ractor_harness_test.rb @@ -0,0 +1,40 @@ +require_relative 'test_helper' +require 'json' +require 'open3' +require 'tmpdir' + +describe 'Ractor harness' do + it 'warms up at the measured Ractor count' do + skip 'target Ruby has no Ractor' unless defined?(Ractor) + root = File.expand_path('..', __dir__) + Dir.mktmpdir do |dir| + result_path = File.join(dir, 'results.json') + calls_path = File.join(dir, 'calls.txt') + script = File.join(dir, 'workload.rb') + File.write(script, <<~RUBY) + require #{File.join(root, 'harness', 'loader').inspect} + run_benchmark(1) do |count| + File.open(#{calls_path.inspect}, 'a') { |f| f.puts(count) } + end + RUBY + env = { + 'RUBYOPT' => nil, + 'RUBYLIB' => nil, + 'BUNDLE_GEMFILE' => nil, + 'RUBY_BENCH_RACTORS' => '2', + 'WARMUP_ITRS' => '2', + 'MIN_BENCH_ITRS' => '1', + 'RESULT_JSON_PATH' => result_path + } + + stdout, stderr, status = Open3.capture3(env, RbConfig.ruby, "-I#{File.join(root, 'harness-ractor')}", script) + + assert status.success?, "workload failed:\n#{stdout}\n#{stderr}" + data = JSON.parse(File.read(result_path)) + assert_equal({ '2' => 2 }, data['warmup_by_ractors'].transform_values(&:size)) + assert_equal({ '2' => 1 }, data['bench_by_ractors'].transform_values(&:size)) + assert_equal data['warmup_by_ractors']['2'], data['warmup'] + assert_equal ['2'] * 6, File.readlines(calls_path, chomp: true), '(2 warmup + 1 measured) iterations x 2 Ractors' + end + end +end diff --git a/test/results_table_builder_test.rb b/test/results_table_builder_test.rb index 97246775..a98764d9 100644 --- a/test/results_table_builder_test.rb +++ b/test/results_table_builder_test.rb @@ -85,16 +85,18 @@ raw = { 'master' => { 'symbol-name-ractor' => { - 'warmup' => [], + 'warmup' => [3.0, 8.0], 'bench' => [1.0, 2.0], + 'warmup_by_ractors' => { '0' => [3.0], '2' => [8.0] }, 'bench_by_ractors' => { '0' => [1.0, 1.0], '2' => [2.0, 2.0] }, 'rss' => 10 * 1024 * 1024 } }, 'exp' => { 'symbol-name-ractor' => { - 'warmup' => [], + 'warmup' => [1.0, 2.0], 'bench' => [0.5, 1.0], + 'warmup_by_ractors' => { '0' => [1.0], '2' => [2.0] }, 'bench_by_ractors' => { '0' => [0.5, 0.5], '2' => [1.0, 1.0] }, 'rss' => 10 * 1024 * 1024 } @@ -119,6 +121,10 @@ assert_equal '', table[2][0] assert_equal '2', table[2][1] + # 1st itr uses each count's first warmup: r=0 3000ms vs 1000ms, r=2 8000ms vs 2000ms + assert_in_delta 3.0, table[1][4], 0.01 + assert_in_delta 4.0, table[2][4], 0.01 + # count=0 row: master 1000ms vs exp 500ms => ratio 2.0 assert_in_delta 2.0, table[1][5].to_f, 0.01 # count=2 row: master 2000ms vs exp 1000ms => ratio 2.0 @@ -860,6 +866,7 @@ def ractor_gc_blob(groups) 'bench' => groups.values.flat_map { |g| g[:bench] }, 'rss' => 10 * 1024 * 1024, 'gc_scope' => 'ractor-local-workload', + 'warmup_by_ractors' => groups.transform_values { [] }, 'bench_by_ractors' => groups.transform_values { |g| g[:bench] }, 'gc_by_ractors' => groups.transform_values { |g| g[:gc] } }