From 08d11eff7016b79dd68bae3798ea0563b547eb6c Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 6 Oct 2026 16:52:10 +0100 Subject: [PATCH 1/4] Move the Ractor GC support check into GCStats These belong with the rest of the Ractor GC sampling helpers. --- harness-ractor/harness.rb | 26 ++------------------------ lib/gc_stats.rb | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/harness-ractor/harness.rb b/harness-ractor/harness.rb index f7c42667..1c73f3b8 100644 --- a/harness-ractor/harness.rb +++ b/harness-ractor/harness.rb @@ -48,7 +48,7 @@ def run_benchmark(num_itrs_hint, ractor_args: [], &block) bench_itrs = MAX_ITERS if bench_itrs > MAX_ITERS if RACTOR_GC_ENABLED - check_ractor_gc_support + GCStats.check_ractor_gc_support! gc_config = GC.config.transform_keys(&:to_s) if GC.respond_to?(:config) GCStats.with_measure_total_time do run_warmup(warmup_itrs, ractor_args, &block) @@ -61,18 +61,6 @@ def run_benchmark(num_itrs_hint, ractor_args: [], &block) end end -def check_ractor_gc_support - unless GCStats.ractor_local_gc_supported? - raise NotImplementedError, "Ractor GC metrics require Ruby 4.1 or newer" - end - unless GC.respond_to?(:total_time) && GC.respond_to?(:measure_total_time) && GC.respond_to?(:measure_total_time=) - raise NotImplementedError, "Ractor GC metrics require GC.total_time and GC.measure_total_time=" - end - unless GCStats.global_gc_attributed? - raise NotImplementedError, "Ractor GC metrics require per-Ractor global GC attribution (ruby/ruby#19147)" - end -end - def run_warmup(warmup_itrs, ractor_args, &block) warmup_itrs.times do args = ractor_args.empty? ? [] : ractor_deep_dup(ractor_args) @@ -110,16 +98,6 @@ def run_benchmark_timing(bench_itrs, ractor_args, &block) return_results([], stats.values.flatten, bench_by_ractors: stats) end -RACTOR_GC_SERIES = { - "gc_count_bench" => "gc_count", - "gc_global_count_bench" => "gc_global_count", - "gc_major_count_bench" => "gc_major_count", - "gc_minor_count_bench" => "gc_minor_count", - "gc_marking_time_bench" => "gc_marking_time", - "gc_sweeping_time_bench" => "gc_sweeping_time", - "gc_total_time_bench" => "gc_total_time_ns", -}.freeze - def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) stats = Hash.new { |h,k| h[k] = [] } gc_by_ractors = {} @@ -144,7 +122,7 @@ def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) agg = GCStats.aggregate(worker_samples) total_ms = agg["gc_total_time_ns"]&.fdiv(1_000_000) - RACTOR_GC_SERIES.each do |series_name, field| + GCStats::RACTOR_SERIES.each do |series_name, field| series[series_name] << (field == "gc_total_time_ns" ? total_ms : agg[field]) end CONTROLLER_GC_SERIES.each_key { |series_name| series[series_name] << controller_deltas[series_name] } diff --git a/lib/gc_stats.rb b/lib/gc_stats.rb index 3a39186a..1106bef3 100644 --- a/lib/gc_stats.rb +++ b/lib/gc_stats.rb @@ -16,6 +16,16 @@ module GCStats SCALAR_FIELD_NAMES = (SCALAR_FIELDS.map(&:first) + [TOTAL_TIME_FIELD]).freeze + RACTOR_SERIES = { + "gc_count_bench" => "gc_count", + "gc_global_count_bench" => "gc_global_count", + "gc_major_count_bench" => "gc_major_count", + "gc_minor_count_bench" => "gc_minor_count", + "gc_marking_time_bench" => "gc_marking_time", + "gc_sweeping_time_bench" => "gc_sweeping_time", + "gc_total_time_bench" => TOTAL_TIME_FIELD, + }.freeze + def stat_available?(key) GC.stat(key).is_a?(Numeric) rescue ArgumentError @@ -48,6 +58,18 @@ def global_gc_attributed? status.success? end + def check_ractor_gc_support! + unless ractor_local_gc_supported? + raise NotImplementedError, "Ractor GC metrics require Ruby 4.1 or newer" + end + unless GC.respond_to?(:total_time) && GC.respond_to?(:measure_total_time) && GC.respond_to?(:measure_total_time=) + raise NotImplementedError, "Ractor GC metrics require GC.total_time and GC.measure_total_time=" + end + unless global_gc_attributed? + raise NotImplementedError, "Ractor GC metrics require per-Ractor global GC attribution (ruby/ruby#19147)" + end + end + def heap_snapshot return {} unless GC.respond_to?(:stat_heap) GC.stat_heap From aa38088ee6a5d70ad0a70a42fa43867337a0dc38 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 11:33:43 +0100 Subject: [PATCH 2/4] Add a scenario mode to harness-ractor The ractor harness runs the benchmark block inside every Ractor it spawns. That doesn't work for benchmarks that need to coordinate their own Ractors from the main Ractor, for example to send them messages, or to keep them alive while we measure memory. With run_benchmark(n, scenario: true) the harness calls the block once per trial on the main Ractor with the Ractor count, and the block spawns its own Ractors. If the block returns a proc, the harness calls it after the memory measurement so idle Ractors can be cleaned up. Scenario mode doesn't run count 0 or a warmup. Each trial records the time, the RSS retained after a full GC compared to a baseline taken before the first trial, and the peak RSS while the block ran. We use GC.start(global: true) when it's available, because otherwise only the main Ractor is collected on Ractor-local GC builds. The harness can't tell which Ractors are the workers, so with --ractor-gc each worker wraps its body in measure_worker_gc and the main Ractor passes the samples to record_worker_gc. --- README.md | 48 ++++++- harness-ractor/harness.rb | 223 +++++++++++++++++++++++++++--- lib/benchmark_runner.rb | 15 +- lib/benchmark_runner/cli.rb | 13 +- test/benchmark_runner_cli_test.rb | 69 +++++++++ test/ractor_gc_harness_test.rb | 126 ++++++++++++++++- 6 files changed, 465 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index db115f02..a7d49513 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,51 @@ intended to be used with any harness except `harness-ractor`. Note: The `harness-ractor` harness is automatically selected when using these categories, so there's no need to specify `--harness` manually. +### Ractor Scenario Benchmarks + +By default, `harness-ractor` spawns the worker Ractors and runs the benchmark +block inside each of them. A benchmark that calls +`run_benchmark(n, scenario: true)` uses scenario mode instead. The harness +calls the block one time per trial in the main Ractor, with the Ractor count as +its argument. The block spawns and coordinates its own Ractors. When the block +returns a proc, the harness calls the proc after the retention measurement. + +Scenario mode skips Ractor count 0 and runs no warmup. For each trial, the +harness records: + +* the time of the block call, which includes all work in the block; +* the retained RSS: the RSS after a full GC, minus the RSS of the process + before the first trial; +* the peak RSS: the highest RSS that the harness reads just before the block + call, every 5 ms during it (`RACTOR_MEM_PEAK_SAMPLE_INTERVAL`), and just + after it. + +The harness prints `BENCH_METRIC retained_mib=` and +`BENCH_METRIC peak_mib=...` lines, plus one pair per ractor count. The JSON +fields `ractor_mem_medians` and `ractor_mem_samples` hold the same data. The +summary table of `run_benchmarks.rb` does not show it. The ractor counts and +trials are controlled with `RUBY_BENCH_RACTORS` (default `1,2,4,6,8`) and +`MIN_BENCH_ITRS` (default: the iteration count that the benchmark passes to +`run_benchmark`). + +The harness collects with `GC.start(global: true)` when the target Ruby's +`GC.start` accepts the `global:` keyword. Some Ruby 4.1 builds do not accept it. +On a target with Ractor-local GC, a plain `GC.start` collects only the main +Ractor's object space. The JSON field `ractor_mem_settle` records `global` or +`default`. + +With `--ractor-gc` (`RUBY_BENCH_RACTOR_GC=1`), the harness cannot see which +Ractors are workers. A scenario wraps each worker body in +`measure_worker_gc { ... }`, which returns `[result, sample]`. The main Ractor +passes each sample to `record_worker_gc(worker_index, sample)`. A trial fails +when its recorded worker indexes are not `0...count`. + +Worker samples cover only the workers' own object spaces during the scenario. +They do not include allocation by the main Ractor, such as the messages that +the main Ractor sends to the workers. They also do not include the GCs that +the harness runs to measure retention. The JSON field `gc_controller_samples` +covers the main Ractor during the scenario. + ## Ruby options By default, ruby-bench benchmarks the Ruby used for `run_benchmarks.rb`. @@ -303,7 +348,8 @@ process's lifetime peak from `getrusage`. ## Measuring Ractor GC activity The `--ractor-gc` option of `run_benchmarks.rb` collects Ractor-local GC -metrics for benchmarks that use the Ractor harness (`--category ractor`). +metrics for benchmarks that use the Ractor harness (`--category ractor`), +in both the per-worker mode and scenario mode. The target must use Ruby 4.1 or newer with per-Ractor global GC attribution ([ruby/ruby#19147](https://github.com/ruby/ruby/pull/19147)); older targets fail before warmup. diff --git a/harness-ractor/harness.rb b/harness-ractor/harness.rb index 1c73f3b8..abce9617 100644 --- a/harness-ractor/harness.rb +++ b/harness-ractor/harness.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true require_relative '../harness/harness-common' +require 'etc' Warning[:experimental] = false ENV["RUBY_BENCH_RACTOR_HARNESS"] = "1" @@ -41,19 +42,31 @@ def join end MAX_ITERS = Integer(ENV.fetch("MAX_BENCH_ITRS", 5)) +WORKER_GC_SAMPLES = [] +SETTLE_SLEEP = Float(ENV.fetch("RACTOR_MEM_SETTLE_SLEEP", 0.1)) +PEAK_SAMPLE_INTERVAL = Float(ENV.fetch("RACTOR_MEM_PEAK_SAMPLE_INTERVAL", 0.005)) +GLOBAL_GC_START = GC.method(:start).parameters.include?([:key, :global]) +PAGE_SIZE = Etc.sysconf(Etc::SC_PAGESIZE) -def run_benchmark(num_itrs_hint, ractor_args: [], &block) +def run_benchmark(num_itrs_hint, ractor_args: [], scenario: false, &block) warmup_itrs = Integer(ENV.fetch('WARMUP_ITRS', 5)) bench_itrs = Integer(ENV.fetch('MIN_BENCH_ITRS', num_itrs_hint)) bench_itrs = MAX_ITERS if bench_itrs > MAX_ITERS + raise ArgumentError, "a scenario benchmark does not take ractor_args" if scenario && !ractor_args.empty? if RACTOR_GC_ENABLED GCStats.check_ractor_gc_support! gc_config = GC.config.transform_keys(&:to_s) if GC.respond_to?(:config) GCStats.with_measure_total_time do - run_warmup(warmup_itrs, ractor_args, &block) - run_benchmark_gc(bench_itrs, Ractor.make_shareable(block), ractor_args, gc_config: gc_config) + if scenario + run_scenario_benchmark(bench_itrs, gc_config: gc_config, &block) + else + run_warmup(warmup_itrs, ractor_args, &block) + run_benchmark_gc(bench_itrs, Ractor.make_shareable(block), ractor_args, gc_config: gc_config) + end end + elsif scenario + run_scenario_benchmark(bench_itrs, gc_config: nil, &block) else puts "r: itr: time" run_warmup(warmup_itrs, ractor_args, &block) @@ -102,10 +115,7 @@ def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) stats = Hash.new { |h,k| h[k] = [] } gc_by_ractors = {} - header = +"r: itr: time gc_total marking sweeping gc_count major minor global" - CONTROLLER_GC_SERIES.each_value { |(_stat_key, label)| header << " %9s" % label } - puts header - puts "(* controller-observed compacting cycles; may overlap global counts and is not additive.)" if CONTROLLER_GC_SERIES.any? + print_gc_header("r: itr: time") RACTORS.each do |rs| group = { "gc_worker_samples" => [] } @@ -120,20 +130,9 @@ def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) group["gc_worker_samples"] << worker_samples group["gc_controller_samples"] << controller_sample if controller_sample - agg = GCStats.aggregate(worker_samples) - total_ms = agg["gc_total_time_ns"]&.fdiv(1_000_000) - GCStats::RACTOR_SERIES.each do |series_name, field| - series[series_name] << (field == "gc_total_time_ns" ? total_ms : agg[field]) - end - CONTROLLER_GC_SERIES.each_key { |series_name| series[series_name] << controller_deltas[series_name] } - + agg = record_gc_series(series, worker_samples, controller_deltas) itr_str = "%-3s %4s %6s" % [rs, "##{num_itrs}:", "#{(1000 * elapsed).to_i}ms"] - itr_str << " %8s" % (total_ms ? "%.1fms" % total_ms : "N/A") - itr_str << " %8s" % (agg["gc_marking_time"] ? "#{agg["gc_marking_time"]}ms" : "N/A") - itr_str << " %8s" % (agg["gc_sweeping_time"] ? "#{agg["gc_sweeping_time"]}ms" : "N/A") - itr_str << " %9s %9s %9s %9s" % [agg["gc_count"], agg["gc_major_count"], agg["gc_minor_count"], agg["gc_global_count"]].map { |v| v.nil? ? "N/A" : v.to_s } - CONTROLLER_GC_SERIES.each_key { |series_name| itr_str << " %9s" % (controller_deltas[series_name] || "N/A") } - puts itr_str + puts itr_str + gc_columns(agg, controller_deltas) end series.each do |name, values| @@ -142,15 +141,44 @@ def run_benchmark_gc(bench_itrs, block, ractor_args, gc_config:) gc_by_ractors[rs] = group end + return_results([], stats.values.flatten, bench_by_ractors: stats, **ractor_gc_results(gc_by_ractors, gc_config)) +end + +def print_gc_header(prefix) + header = prefix + " gc_total marking sweeping gc_count major minor global" + CONTROLLER_GC_SERIES.each_value { |(_stat_key, label)| header << " %9s" % label } + puts header + puts "(* controller-observed compacting cycles; may overlap global counts and is not additive.)" if CONTROLLER_GC_SERIES.any? +end + +def record_gc_series(series, worker_samples, controller_deltas) + agg = GCStats.aggregate(worker_samples) + GCStats::RACTOR_SERIES.each do |series_name, field| + series[series_name] << (field == GCStats::TOTAL_TIME_FIELD ? agg[field]&.fdiv(1_000_000) : agg[field]) + end + CONTROLLER_GC_SERIES.each_key { |series_name| series[series_name] << controller_deltas[series_name] } + agg +end + +def gc_columns(agg, controller_deltas) + total_ms = agg[GCStats::TOTAL_TIME_FIELD]&.fdiv(1_000_000) + columns = " %8s" % (total_ms ? "%.1fms" % total_ms : "N/A") + columns << " %8s" % (agg["gc_marking_time"] ? "#{agg["gc_marking_time"]}ms" : "N/A") + columns << " %8s" % (agg["gc_sweeping_time"] ? "#{agg["gc_sweeping_time"]}ms" : "N/A") + columns << " %9s %9s %9s %9s" % [agg["gc_count"], agg["gc_major_count"], agg["gc_minor_count"], agg["gc_global_count"]].map { |v| v.nil? ? "N/A" : v.to_s } + CONTROLLER_GC_SERIES.each_key { |series_name| columns << " %9s" % (controller_deltas[series_name] || "N/A") } + columns +end + +def ractor_gc_results(gc_by_ractors, gc_config) extra = { - bench_by_ractors: stats, gc_scope: "ractor-local-workload", gc_stat_scope: "ractor-local", gc_measure_total_time_scope: "ractor-local", gc_by_ractors: gc_by_ractors, } extra[:gc_config] = gc_config if gc_config - return_results([], stats.values.flatten, **extra) + extra end def controller_gc_snapshot @@ -203,6 +231,157 @@ def run_controller_gc_iteration(ractor_args, &block) [elapsed, [sample], nil, controller_gc_deltas(counters_before, counters_after)] end +def measure_worker_gc + return [yield, nil] unless RACTOR_GC_ENABLED + + result = nil + sample = GCStats.measure { result = yield } + [result, sample] +end + +def record_worker_gc(worker_index, sample) + return unless sample + + WORKER_GC_SAMPLES << sample.merge("worker_index" => worker_index) +end + +def run_scenario_benchmark(bench_itrs, gc_config:, &scenario) + counts = RACTORS - [0] + raise ArgumentError, "a scenario benchmark needs a Ractor count above 0" if counts.empty? + bench_itrs = 1 if bench_itrs < 1 + + gc_settle + base_rss = get_rss + puts "base RSS: #{format_mib(base_rss)}" + header = "r: itr: time" + " %9s %9s" % ["retained", "peak"] + RACTOR_GC_ENABLED ? print_gc_header(header) : puts(header) + + times = Hash.new { |h, k| h[k] = [] } + memory = Hash.new { |h, k| h[k] = { retained: [], peak: [] } } + gc_by_ractors = {} + + counts.each do |count| + group = { "gc_worker_samples" => [], "gc_controller_samples" => [] } + series = Hash.new { |h, k| h[k] = [] } + + bench_itrs.times do |itr| + elapsed, finish, peak, gc = run_scenario_iteration(count, &scenario) + gc_settle + retained = get_rss - base_rss + finish&.call + gc_settle + + times[count] << elapsed + memory[count][:retained] << retained + memory[count][:peak] << peak + itr_str = "%-3s %4s %6s %9s %9s" % [count, "##{itr + 1}:", "#{(1000 * elapsed).to_i}ms", format_mib(retained), format_mib(peak)] + if gc + group["gc_worker_samples"] << gc[:workers] + group["gc_controller_samples"] << gc[:controller] + itr_str << gc_columns(record_gc_series(series, gc[:workers], gc[:deltas]), gc[:deltas]) + end + puts itr_str + end + + gc_by_ractors[count] = group.merge(series) + end + + medians = counts.to_h do |count| + [count, { retained: Stats.new(memory[count][:retained]).median, peak: Stats.new(memory[count][:peak]).median }] + end + print_memory_metrics(medians) + + extra = { + bench_by_ractors: times, + ractor_mode: "scenario", + ractor_mem_settle: GLOBAL_GC_START ? "global" : "default", + ractor_mem_base_rss: base_rss, + ractor_mem_medians: medians, + ractor_mem_samples: memory, + } + extra.merge!(ractor_gc_results(gc_by_ractors, gc_config)) if RACTOR_GC_ENABLED + return_results([], times.values.flatten, **extra) +end + +def run_scenario_iteration(count, &scenario) + WORKER_GC_SAMPLES.clear + if RACTOR_GC_ENABLED + controller_before = GCStats.snapshot + counters_before = controller_gc_snapshot + end + (finish, elapsed), peak = measure_peak_rss do + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = scenario.call(count) + [result, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started] + end + return [elapsed, finish, peak, nil] unless RACTOR_GC_ENABLED + + counters_after = controller_gc_snapshot + gc = { + controller: GCStats.delta(controller_before, GCStats.snapshot), + deltas: controller_gc_deltas(counters_before, counters_after), + workers: scenario_worker_samples(count), + } + [elapsed, finish, peak, gc] +end + +def scenario_worker_samples(count) + workers = WORKER_GC_SAMPLES.sort_by { |sample| sample["worker_index"] } + indexes = workers.map { |sample| sample["worker_index"] } + raise "scenario recorded worker GC samples #{indexes.inspect} for #{count} ractors" unless indexes == (0...count).to_a + workers +end + +def gc_settle + 2.times do + if GLOBAL_GC_START + GC.start(full_mark: true, immediate_sweep: true, global: true) + else + GC.start(full_mark: true, immediate_sweep: true) + end + end + sleep SETTLE_SLEEP +end + +def statm_rss + PAGE_SIZE * Integer(File.read("/proc/self/statm").split(" ")[1]) +end + +def measure_peak_rss + stop = false + peak = statm_rss + sampler = Thread.new do + Thread.current.report_on_exception = false + until stop + rss = statm_rss + peak = rss if rss > peak + sleep PEAK_SAMPLE_INTERVAL + end + end + begin + result = yield + ensure + stop = true + sampler.join + end + [result, [peak, statm_rss].max] +end + +def format_mib(bytes) + "%.1fMiB" % (bytes / 2**20.0) +end + +def print_memory_metrics(medians) + medians.each do |count, m| + puts format("BENCH_METRIC retained_mib_r%d=%.1f", count, m[:retained] / 2**20.0) + puts format("BENCH_METRIC peak_mib_r%d=%.1f", count, m[:peak] / 2**20.0) + end + worst_count, worst = medians.max_by { |_count, m| m[:retained] } + puts format("BENCH_METRIC retained_mib=%.1f", worst[:retained] / 2**20.0) + puts format("BENCH_METRIC peak_mib=%.1f", medians.values.map { |m| m[:peak] }.max / 2**20.0) + puts format("BENCH_METRIC worst_ractor_count=%d", worst_count) +end + # NOTE: we use `ractor_deep_dup` instead of `Ractor.make_shareable(copy: true)` for the case of # sending args to the block without a ractor because the arguments passed to `run_benchmark` are # sometimes modified, and we want to allow that because it improves compatibility. We don't want diff --git a/lib/benchmark_runner.rb b/lib/benchmark_runner.rb index 38f9222e..d8485dbb 100644 --- a/lib/benchmark_runner.rb +++ b/lib/benchmark_runner.rb @@ -96,12 +96,21 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_ output_str << "- controller compacts/iter*: the main Ractor's GC.stat(:compact_count) delta per iteration. Every global compacting cycle increments compact_count in every object space. Do not sum it across workers.#{other_names.empty? ? '' : " Comparison tables show #{base_name} → comparison values."}\n" end - if sections.any? { |section| section[:gc_scope] == 'ractor-local-workload' && section[:gc_table] } + ractor_gc_sections = sections.select { |section| section[:gc_scope] == 'ractor-local-workload' && section[:gc_table] } + unless ractor_gc_sections.empty? + modes = ractor_gc_sections.flat_map { |section| section.fetch(:ractor_gc_modes, []) } + worker_mode = modes.include?(:worker) output_str << "Ractor GC scope note:\n" - scope_columns = +"- (worker sum) columns add Ractor-local counters across the sampled workers of each iteration; the main Ractor performs the count-0 workload." + scope_columns = +"- (worker sum) columns add Ractor-local counters across the sampled workers of each iteration" + scope_columns << (worker_mode ? "; the main Ractor performs the count-0 workload of per-worker benchmarks." : ".") scope_columns << " GC ms/worker divides each iteration's worker-sum GC time by its sampled worker count, then averages." if other_names.empty? output_str << "#{scope_columns} Controller snapshots and per-worker heap detail are in the JSON output, not this table.\n" - output_str << "- Ruby's Ractor-retirement GC (after a worker's stack is torn down) and Ractors created by the workload itself are not sampled.\n" + output_str << "- Ruby's Ractor-retirement GC (after a worker's stack is torn down) is not sampled." + output_str << " Ractors created by the workload of a per-worker benchmark are not sampled." if worker_mode + output_str << "\n" + if modes.include?(:scenario) + output_str << "- Scenario benchmarks sample only the worker Ractors whose bodies the scenario wraps in measure_worker_gc; other Ractors that a scenario spawns are not sampled. Main-Ractor allocation during the scenario and the retention-measurement GCs are not in the worker sums; gc_controller_samples in the JSON output cover the main Ractor during the scenario.\n" + end output_str << "- GC time is CPU-time accounting, not elapsed pause time; summed across Ractors it can exceed wall time.\n" output_str << "- Per-GC ratios divide by recorded GC counts, not complete process-wide GC cycles. Phase times are integer milliseconds; total GC time is kept at nanosecond resolution in the raw worker samples.\n" end diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index ed59d2d1..efbcaad9 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -204,10 +204,21 @@ def build_output_section(executable_names, bench_data, bench_failures, harness, gc_table: gc_table, gc_format: gc_format, } - section[:gc_scope] = 'ractor-local-workload' if ResultsTableBuilder.ractor_gc_data?(section_data) + if ResultsTableBuilder.ractor_gc_data?(section_data) + section[:gc_scope] = 'ractor-local-workload' + section[:ractor_gc_modes] = ractor_gc_modes(section_data) + end section end + def ractor_gc_modes(section_data) + blobs = section_data.values.flat_map(&:values) + blobs.filter_map do |blob| + next unless blob.is_a?(Hash) && blob['gc_scope'] == 'ractor-local-workload' + blob['ractor_mode'] == 'scenario' ? :scenario : :worker + end.uniq + end + def sorted_benchmark_names(executable_names, bench_data) builder = ResultsTableBuilder.new( executable_names: executable_names, diff --git a/test/benchmark_runner_cli_test.rb b/test/benchmark_runner_cli_test.rb index 7baf82f3..5e19a50b 100644 --- a/test/benchmark_runner_cli_test.rb +++ b/test/benchmark_runner_cli_test.rb @@ -195,6 +195,75 @@ def create_args(overrides = {}) assert_equal 'ractor-local-workload', section[:gc_scope], 'the raw blob scope must tag the section; expansion drops gc_scope from counts without GC data' end + + it 'describes scenario sampling and omits per-worker notes for a scenario-only section' do + args = create_args + cli = BenchmarkRunner::CLI.new(args) + gc_group = { + 'gc_count_bench' => [3], + 'gc_major_count_bench' => [1], + 'gc_minor_count_bench' => [2], + 'gc_marking_time_bench' => [1.0], + 'gc_sweeping_time_bench' => [1.0], + 'gc_total_time_bench' => [2.0], + 'gc_worker_samples' => [[{ 'gc_count' => 3, 'worker_index' => 0 }]] + } + bench_data = { + 'ruby' => { + 'ractor-dead-set' => { + 'warmup' => [], + 'bench' => [1.0, 2.0], + 'rss' => 10 * 1024 * 1024, + 'gc_scope' => 'ractor-local-workload', + 'ractor_mode' => 'scenario', + 'bench_by_ractors' => { '1' => [1.0], '2' => [2.0] }, + 'gc_by_ractors' => { '1' => gc_group, '2' => gc_group } + } + } + } + + sections = cli.send(:build_output_sections, ['ruby'], bench_data, { 'ractor-dead-set' => 'harness-ractor' }, {}) + + output = BenchmarkRunner.build_output_text({ 'ruby' => 'ruby 4.1.0dev' }, nil, nil, {}, sections: sections) + assert_match(/Scenario benchmarks sample only the worker Ractors whose bodies the scenario wraps in measure_worker_gc/, output) + refute_match(/created by the workload of a per-worker benchmark/, output) + refute_match(/count-0 workload/, output) + end + + it 'prints both per-worker and scenario notes for a section that mixes the two modes' do + args = create_args + cli = BenchmarkRunner::CLI.new(args) + gc_group = { + 'gc_count_bench' => [3], + 'gc_major_count_bench' => [1], + 'gc_minor_count_bench' => [2], + 'gc_total_time_bench' => [2.0], + 'gc_worker_samples' => [[{ 'gc_count' => 3, 'worker_index' => 0 }]] + } + blob = ->(counts, mode) do + data = { + 'warmup' => [], + 'bench' => counts.map { 1.0 }, + 'rss' => 10 * 1024 * 1024, + 'gc_scope' => 'ractor-local-workload', + 'bench_by_ractors' => counts.to_h { |c| [c, [1.0]] }, + 'gc_by_ractors' => counts.to_h { |c| [c, gc_group] } + } + data['ractor_mode'] = mode if mode + data + end + bench_data = { 'ruby' => { 'object-new' => blob.call(%w[0 2], nil), 'ractor-dead-set' => blob.call(%w[1 2], 'scenario') } } + harnesses = { 'object-new' => 'harness-ractor', 'ractor-dead-set' => 'harness-ractor' } + + sections = cli.send(:build_output_sections, ['ruby'], bench_data, harnesses, {}) + + assert_equal 1, sections.size + assert_equal [:scenario, :worker], sections.first[:ractor_gc_modes].sort + output = BenchmarkRunner.build_output_text({ 'ruby' => 'ruby 4.1.0dev' }, nil, nil, {}, sections: sections) + assert_match(/count-0 workload of per-worker benchmarks/, output) + assert_match(/Ractors created by the workload of a per-worker benchmark are not sampled/, output) + assert_match(/Scenario benchmarks sample only the worker Ractors/, output) + end end describe '#run integration test' do diff --git a/test/ractor_gc_harness_test.rb b/test/ractor_gc_harness_test.rb index 7ad82e38..b3460fee 100644 --- a/test/ractor_gc_harness_test.rb +++ b/test/ractor_gc_harness_test.rb @@ -103,6 +103,60 @@ def GCStats.global_gc_attributed?(*) = false puts "workload_ran=#{workload_ran}" RUBY + MEM_WORKLOAD_BODY = <<~'RUBY' + run_benchmark(2, scenario: true) do |count| + workers = count.times.map do |worker| + Ractor.new(worker) do |worker_id| + measure_worker_gc do + 20_000.times { Object.new } + GC.start(full_mark: true, immediate_sweep: true) + worker_id + end + end + end + workers.each_with_index do |worker, worker_id| + _, sample = worker.value + record_worker_gc(worker_id, sample) + end + nil + end + RUBY + + MEM_UNRECORDED_BODY = <<~'RUBY' + run_benchmark(2, scenario: true) do |count| + count.times.map { Ractor.new { 10_000.times { Object.new } } }.each(&:value) + nil + end + RUBY + + QUICK_SCENARIO_BODY = <<~'RUBY' + run_benchmark(1, scenario: true) do |_count| + Array.new(1_000) { |i| "item #{i}" } + nil + end + RUBY + + SLEEPING_SCENARIO_BODY = <<~'RUBY' + run_benchmark(1, scenario: true) do |_count| + sleep 0.01 + nil + end + RUBY + + FINISH_SCENARIO_BODY = <<~'RUBY' + run_benchmark(2, scenario: true) do |count| + proc { puts "finish called for #{count}" } + end + RUBY + + SCENARIO_NO_GC_ENV = { + 'RUBY_BENCH_RACTOR_GC' => nil, + 'RUBY_BENCH_RACTORS' => '1', + 'MIN_BENCH_ITRS' => '1', + 'MAX_BENCH_ITRS' => '1', + 'RACTOR_MEM_SETTLE_SLEEP' => '0', + }.freeze + before do @explicit_target = !ENV['RACTOR_GC_TEST_RUBY'].nil? @ruby = ENV['RACTOR_GC_TEST_RUBY'] || RbConfig.ruby @@ -132,7 +186,7 @@ def GCStats.global_gc_attributed?(*) = false end end - def run_workload(body) + def run_workload(body, env: {}) Dir.mktmpdir do |dir| result_path = File.join(dir, 'results.json') script = File.join(dir, 'workload.rb') @@ -145,7 +199,7 @@ def run_workload(body) 'MAX_BENCH_ITRS' => '2', 'MIN_BENCH_TIME' => '0', 'RESULT_JSON_PATH' => result_path - ) + ).merge(env) stdout, stderr, status = Open3.capture3(env, @ruby, "-I#{File.join(ROOT, 'harness-ractor')}", script, chdir: ROOT) yield stdout, stderr, status, result_path end @@ -294,6 +348,74 @@ def run_workload(body) end end + it 'collects per-worker GC samples from a scenario benchmark' do + run_workload(MEM_WORKLOAD_BODY) do |stdout, stderr, status, result_path| + assert status.success?, "workload failed:\n#{stdout}\n#{stderr}" + + data = JSON.parse(File.read(result_path)) + assert_equal 'ractor-local-workload', data['gc_scope'] + assert_equal 'scenario', data['ractor_mode'] + assert_equal 'global', data['ractor_mem_settle'] + assert_equal %w[1 2], data['bench_by_ractors'].keys.sort, 'scenario mode skips count 0' + assert_equal %w[1 2], data['gc_by_ractors'].keys.sort + assert_equal %w[1 2], data['ractor_mem_samples'].keys.sort + data['ractor_mem_samples'].each do |count, samples| + assert_equal 2, samples['retained'].length, "count #{count} retained samples" + assert_equal 2, samples['peak'].length, "count #{count} peak samples" + end + + data['gc_by_ractors'].each do |count, group| + assert_equal 2, data['bench_by_ractors'][count].length, "count #{count} timing samples" + assert_equal 2, group['gc_worker_samples'].length, "count #{count} measured iterations" + assert_equal 2, group['gc_controller_samples'].length, "count #{count} controller samples" + group['gc_worker_samples'].each_with_index do |workers, i| + assert_equal (0...count.to_i).to_a, workers.map { |w| w['worker_index'] }, 'spawn-index order' + workers.each { |w| assert_operator w['gc_count'], :>, 0, 'every worker records GC activity' } + assert_equal workers.sum { |w| w['gc_count'] }, group['gc_count_bench'][i] + assert_in_delta workers.sum { |w| w['gc_total_time_ns'] } / 1_000_000.0, group['gc_total_time_bench'][i], 1e-9 + end + end + end + end + + it 'records a peak RSS for a scenario that returns before the sampler thread runs' do + run_workload(QUICK_SCENARIO_BODY, env: SCENARIO_NO_GC_ENV) do |stdout, stderr, status, result_path| + assert status.success?, "workload failed:\n#{stdout}\n#{stderr}" + + data = JSON.parse(File.read(result_path)) + refute data.key?('gc_by_ractors'), 'scenario mode without --ractor-gc writes no GC data' + peaks = data.dig('ractor_mem_samples', '1', 'peak') + assert_equal 1, peaks.length + assert_operator peaks.first, :>=, data['ractor_mem_base_rss'] / 2, 'the peak is a real RSS reading, not the 0 start value' + end + end + + it 'excludes the peak sampler shutdown from the scenario time' do + env = SCENARIO_NO_GC_ENV.merge('RACTOR_MEM_PEAK_SAMPLE_INTERVAL' => '0.5') + run_workload(SLEEPING_SCENARIO_BODY, env: env) do |stdout, stderr, status, result_path| + assert status.success?, "workload failed:\n#{stdout}\n#{stderr}" + + time = JSON.parse(File.read(result_path)).dig('bench_by_ractors', '1').first + assert_operator time, :<, 0.25, 'the time must not wait for the sampler to finish its 0.5 s sleep' + end + end + + it 'calls the proc that a scenario returns once per trial' do + env = SCENARIO_NO_GC_ENV.merge('RUBY_BENCH_RACTORS' => '1,2', 'MIN_BENCH_ITRS' => '2', 'MAX_BENCH_ITRS' => '2') + run_workload(FINISH_SCENARIO_BODY, env: env) do |stdout, stderr, status, _result_path| + assert status.success?, "workload failed:\n#{stdout}\n#{stderr}" + assert_equal ['finish called for 1'] * 2 + ['finish called for 2'] * 2, stdout.scan(/finish called for \d/) + end + end + + it 'fails a scenario benchmark that does not record its worker GC samples' do + run_workload(MEM_UNRECORDED_BODY) do |stdout, stderr, status, result_path| + refute status.success?, "expected an unrecorded scenario to fail:\n#{stdout}\n#{stderr}" + assert_includes stderr, 'scenario recorded worker GC samples [] for 1 ractors' + refute File.exist?(result_path), 'no results file may be written for a failed benchmark' + end + end + it 'propagates a worker failure without writing partial or dummy results' do run_workload(FAILING_WORKLOAD_BODY) do |stdout, stderr, status, result_path| refute status.success?, "expected worker failure to fail the run:\n#{stdout}\n#{stderr}" From 277eedde626aec4edfebaa2f6e11c97f96d7cd21 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 11:34:42 +0100 Subject: [PATCH 3/4] Add Ractor memory pathology benchmarks These use the scenario mode to measure how much memory we fail to reclaim when multiple Ractors are involved. - ractor-dead-set: each Ractor builds a large live set and exits. - ractor-idle-garbage: each Ractor builds a large set, drops it and then sits idle, so nothing sweeps it. - ractor-msg-backlog: the main Ractor floods sleeping consumers with unshareable messages, so every consumer holds its own copy. --- README.md | 28 +++++++++--- benchmarks.yml | 15 +++++++ benchmarks/ractor-dead-set/benchmark.rb | 34 +++++++++++++++ benchmarks/ractor-idle-garbage/benchmark.rb | 46 ++++++++++++++++++++ benchmarks/ractor-msg-backlog/benchmark.rb | 47 +++++++++++++++++++++ test/ractor_gc_harness_test.rb | 32 ++++++++++++++ 6 files changed, 196 insertions(+), 6 deletions(-) create mode 100644 benchmarks/ractor-dead-set/benchmark.rb create mode 100644 benchmarks/ractor-idle-garbage/benchmark.rb create mode 100644 benchmarks/ractor-msg-backlog/benchmark.rb diff --git a/README.md b/README.md index a7d49513..c2ed9709 100644 --- a/README.md +++ b/README.md @@ -184,13 +184,29 @@ harness records: call, every 5 ms during it (`RACTOR_MEM_PEAK_SAMPLE_INTERVAL`), and just after it. +Three benchmarks use scenario mode to measure pathological memory behaviour +with multiple ractors, for GC work that reclaims ractor-local memory: + +* **`ractor-dead-set`** - Every ractor builds a large live set and terminates. + Retention shows how much of the dead ractors' final live sets a full GC + leaves resident. +* **`ractor-idle-garbage`** - Every ractor builds a large set, drops all + references, then idles without allocating. The garbage cannot be swept + while the ractor idles. +* **`ractor-msg-backlog`** - Unshareable payloads flood the queues of gated + consumer ractors, duplicating the payload data per consumer. Its time + includes the gate sleep (`RACTOR_BACKLOG_GATE_SLEEP`, default 1 second). + +```bash +ruby -Iharness-ractor benchmarks/ractor-dead-set/benchmark.rb +``` + The harness prints `BENCH_METRIC retained_mib=` and `BENCH_METRIC peak_mib=...` lines, plus one pair per ractor count. The JSON fields `ractor_mem_medians` and `ractor_mem_samples` hold the same data. The summary table of `run_benchmarks.rb` does not show it. The ractor counts and trials are controlled with `RUBY_BENCH_RACTORS` (default `1,2,4,6,8`) and -`MIN_BENCH_ITRS` (default: the iteration count that the benchmark passes to -`run_benchmark`). +`MIN_BENCH_ITRS` (default 3 for these benchmarks). The harness collects with `GC.start(global: true)` when the target Ruby's `GC.start` accepts the `global:` keyword. Some Ruby 4.1 builds do not accept it. @@ -205,10 +221,10 @@ passes each sample to `record_worker_gc(worker_index, sample)`. A trial fails when its recorded worker indexes are not `0...count`. Worker samples cover only the workers' own object spaces during the scenario. -They do not include allocation by the main Ractor, such as the messages that -the main Ractor sends to the workers. They also do not include the GCs that -the harness runs to measure retention. The JSON field `gc_controller_samples` -covers the main Ractor during the scenario. +They do not include allocation by the main Ractor, such as the payloads that +`ractor-msg-backlog` sends. They also do not include the GCs that the harness +runs to measure retention. The JSON field `gc_controller_samples` covers the +main Ractor during the scenario. ## Ruby options diff --git a/benchmarks.yml b/benchmarks.yml index 154aefaa..78cae16d 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -296,6 +296,21 @@ json_parse_string: ractor: true ractor_only: true default_harness: harness-ractor +ractor-dead-set: + desc: multiple ractors each build a large live set and terminate, then memory retention after their death is measured. + ractor: true + ractor_only: true + default_harness: harness-ractor +ractor-idle-garbage: + desc: multiple ractors each build and drop a large garbage set, then idle uncollected while retention is measured. + ractor: true + ractor_only: true + default_harness: harness-ractor +ractor-msg-backlog: + desc: unshareable message payloads flood the queues of gated consumer ractors, duplicating data per consumer and driving peak RSS and retention. + ractor: true + ractor_only: true + default_harness: harness-ractor symbol-name-ractor: desc: repeatedly calls Symbol#name on a static symbol under the ractor harness to stress ID-to-string lookup. ractor: true diff --git a/benchmarks/ractor-dead-set/benchmark.rb b/benchmarks/ractor-dead-set/benchmark.rb new file mode 100644 index 00000000..adebb105 --- /dev/null +++ b/benchmarks/ractor-dead-set/benchmark.rb @@ -0,0 +1,34 @@ +# Multiple ractors each build a large live set and then terminate. +# The final live sets of dead ractors are garbage after death. Measured +# retention shows how much of that memory a full GC fails to reclaim. + +Warning[:experimental] = false + +require_relative "../../harness/loader" + +DEAD_SET_ITEMS = Integer(ENV.fetch("RACTOR_DEAD_SET_ITEMS", 100_000)) + +run_benchmark(3, scenario: true) do |num_ractors| + workers = num_ractors.times.map do |worker| + Ractor.new(worker, DEAD_SET_ITEMS) do |worker_id, items| + measure_worker_gc do + keep = [] + i = 0 + while i < items + keep << "worker #{worker_id} item #{i} " + ("y" * 100) + i += 1 + end + keep.size + end + end + end + + total = 0 + workers.each_with_index do |worker, worker_id| + size, sample = worker.value + record_worker_gc(worker_id, sample) + total += size + end + raise "unexpected dead-set size" unless total == num_ractors * DEAD_SET_ITEMS + nil +end diff --git a/benchmarks/ractor-idle-garbage/benchmark.rb b/benchmarks/ractor-idle-garbage/benchmark.rb new file mode 100644 index 00000000..36477403 --- /dev/null +++ b/benchmarks/ractor-idle-garbage/benchmark.rb @@ -0,0 +1,46 @@ +# Multiple ractors each build a large live set, drop all references, then park +# on Ractor.receive without allocating again. Their garbage cannot be swept by +# the main Ractor's GC while they idle, so it is retained until each worker +# is released. The scenario returns a cleanup proc that the harness calls +# after the retention measurement. + +Warning[:experimental] = false + +require_relative "../../harness/loader" + +IDLE_GARBAGE_ITEMS = Integer(ENV.fetch("RACTOR_IDLE_GARBAGE_ITEMS", 100_000)) + +run_benchmark(3, scenario: true) do |num_ractors| + drained = Ractor.new(num_ractors) do |count| + Array.new(count) { Ractor.receive } + end + + workers = num_ractors.times.map do |worker| + Ractor.new(drained, worker, IDLE_GARBAGE_ITEMS) do |ack, worker_id, items| + keep = [] + _, sample = measure_worker_gc do + i = 0 + while i < items + keep << "worker #{worker_id} garbage #{i} " + ("y" * 100) + i += 1 + end + end + message = Ractor.make_shareable([worker_id, sample]) + keep = nil + ack.send message + Ractor.receive + :worker_done + end + end + + ractor, drained_workers = Ractor.select(drained, *workers) + raise "unexpected drain barrier result" unless ractor.equal?(drained) && drained_workers.size == num_ractors + drained_workers.each { |worker_id, sample| record_worker_gc(worker_id, sample) } + + proc do + workers.each do |worker| + worker.send :stop + raise "unexpected worker result" unless worker.value == :worker_done + end + end +end diff --git a/benchmarks/ractor-msg-backlog/benchmark.rb b/benchmarks/ractor-msg-backlog/benchmark.rb new file mode 100644 index 00000000..f1983661 --- /dev/null +++ b/benchmarks/ractor-msg-backlog/benchmark.rb @@ -0,0 +1,47 @@ +# The main Ractor floods the incoming queues of multiple gated consumer +# ractors with unshareable string payloads. The copies made on send pile up +# in the queues while the consumers sleep, which duplicates the payload data +# per consumer and drives peak RSS. Measured retention after the queues are +# drained shows how much of the copied memory a full GC fails to reclaim. + +Warning[:experimental] = false + +require_relative "../../harness/loader" + +BACKLOG_MESSAGES = Integer(ENV.fetch("RACTOR_BACKLOG_MESSAGES", 20_000)) +BACKLOG_GATE_SLEEP = Float(ENV.fetch("RACTOR_BACKLOG_GATE_SLEEP", 1.0)) + +run_benchmark(3, scenario: true) do |num_ractors| + consumers = num_ractors.times.map do |consumer| + Ractor.new(consumer, BACKLOG_GATE_SLEEP) do |consumer_id, gate| + measure_worker_gc do + sleep gate + taken = 0 + loop do + message = Ractor.receive + break if message == :done + taken += 1 + end + taken + end + end + end + + consumers.each_with_index do |consumer, consumer_id| + i = 0 + while i < BACKLOG_MESSAGES + consumer.send "consumer #{consumer_id} message #{i} " + ("x" * 200) + i += 1 + end + consumer.send :done + end + + total = 0 + consumers.each_with_index do |consumer, consumer_id| + taken, sample = consumer.value + record_worker_gc(consumer_id, sample) + total += taken + end + raise "unexpected backlog drain" unless total == num_ractors * BACKLOG_MESSAGES + nil +end diff --git a/test/ractor_gc_harness_test.rb b/test/ractor_gc_harness_test.rb index b3460fee..62cd84fa 100644 --- a/test/ractor_gc_harness_test.rb +++ b/test/ractor_gc_harness_test.rb @@ -416,6 +416,38 @@ def run_workload(body, env: {}) end end + it 'reports every worker of each scenario benchmark' do + %w[ractor-dead-set ractor-idle-garbage ractor-msg-backlog].each do |bench| + Dir.mktmpdir do |dir| + result_path = File.join(dir, 'results.json') + env = CLEAN_ENV.merge( + 'RUBY_BENCH_RACTOR_GC' => '1', + 'RUBY_BENCH_RACTORS' => '1,2', + 'MIN_BENCH_ITRS' => '1', + 'MAX_BENCH_ITRS' => '1', + 'RACTOR_MEM_SETTLE_SLEEP' => '0', + 'RACTOR_DEAD_SET_ITEMS' => '5000', + 'RACTOR_IDLE_GARBAGE_ITEMS' => '5000', + 'RACTOR_BACKLOG_MESSAGES' => '500', + 'RACTOR_BACKLOG_GATE_SLEEP' => '0.05', + 'RESULT_JSON_PATH' => result_path + ) + script = File.join(ROOT, 'benchmarks', bench, 'benchmark.rb') + stdout, stderr, status = Open3.capture3(env, @ruby, "-I#{File.join(ROOT, 'harness-ractor')}", script, chdir: ROOT) + assert status.success?, "#{bench} failed:\n#{stdout}\n#{stderr}" + + data = JSON.parse(File.read(result_path)) + assert_equal %w[1 2], data['gc_by_ractors'].keys.sort, bench + data['gc_by_ractors'].each do |count, group| + assert_equal 1, group['gc_controller_samples'].length, "#{bench} count #{count} controller samples" + workers = group['gc_worker_samples'].fetch(0) + assert_equal (0...count.to_i).to_a, workers.map { |w| w['worker_index'] }, "#{bench} count #{count} workers" + workers.each { |w| assert_kind_of Integer, w['gc_count'], "#{bench} count #{count} worker gc_count" } + end + end + end + end + it 'propagates a worker failure without writing partial or dummy results' do run_workload(FAILING_WORKLOAD_BODY) do |stdout, stderr, status, result_path| refute status.success?, "expected worker failure to fail the run:\n#{stdout}\n#{stderr}" From 085ad51087bc75f647aff340e79b6d52382e4560 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 11:35:54 +0100 Subject: [PATCH 4/4] Split the GC summary into ratio and count tables With Ractor GC data, the comparison GC summary was over 250 characters wide. This commit splits it into a GC time ratios table and a GC counts table. The "ratio" and "worker sum" labels move out of the column headers and into the table titles. Columns with no data in any row are hidden and listed under the table. global/iter is now shown as a count instead of a ratio. It's a GC count, and the legend says the ratios are GC time. --- README.md | 16 ++- lib/benchmark_runner.rb | 39 +++--- lib/benchmark_runner/cli.rb | 9 +- lib/results_table_builder.rb | 184 ++++++++++++++---------- test/benchmark_runner_cli_test.rb | 14 +- test/benchmark_runner_test.rb | 105 +++++++++----- test/results_table_builder_test.rb | 217 ++++++++++++++++++----------- 7 files changed, 366 insertions(+), 218 deletions(-) diff --git a/README.md b/README.md index c2ed9709..be0e46dc 100644 --- a/README.md +++ b/README.md @@ -379,15 +379,21 @@ Ractor's own object space. The JSON output records the scope as `gc_scope: "ractor-local-workload"`, `gc_stat_scope: "ractor-local"`, and `gc_measure_total_time_scope: "ractor-local"`, plus the target's `gc_config`. -The summary table adds these columns: - -* `(worker sum)` columns add the Ractor-local counters of the sampled - workers of each iteration. `GCs/iter` is the sum of `minor/iter`, +The text summary shows GC data in separate tables after the timing table. +A single-executable report has one `GC summary` table. A comparison report +has a `GC time ratios` table (base/comparison) and a `GC counts` table +(base → comparison). A table hides a column that has no data in any row and +lists the hidden columns below the table. A ratio column has no data when it +is `N/A` in every row; a `0.000` ratio stays visible. Any other column has no +data when it is zero or `N/A` in every row. + +* Tables marked `worker sum` add the Ractor-local counters and GC times of + the sampled workers of each iteration. `GCs/iter` is the sum of `minor/iter`, `major/iter`, and `global/iter`; a global cycle counts under `global` on the Ractor that initiated it, not under `major`. Single-executable reports also show `GC ms/worker`, which divides each iteration's worker-sum GC time by its sampled worker count, then averages. -* `controller compacts/iter*` shows the main Ractor's +* `compacts*` shows the main Ractor's `GC.stat(:compact_count)` delta. Every global compacting cycle increments it in every object space, so it is not summed across workers. diff --git a/lib/benchmark_runner.rb b/lib/benchmark_runner.rb index d8485dbb..4b931d22 100644 --- a/lib/benchmark_runner.rb +++ b/lib/benchmark_runner.rb @@ -48,7 +48,7 @@ def write_csv(output_path, ruby_descriptions, table) end # Build output text string with metadata, table, and legend - def build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: false, include_gc: false, include_pvalue: false, gc_table: nil, gc_format: nil, sections: nil) + def build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: false, include_gc: false, include_pvalue: false, gc_tables: nil, sections: nil) base_name, *other_names = ruby_descriptions.keys output_str = +"" @@ -58,16 +58,23 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_ end output_str << "\n" - sections ||= [{ table: table, format: format, failures: bench_failures, include_gc: include_gc, gc_table: gc_table, gc_format: gc_format }] - has_gc_summary = sections.any? { |section| section[:include_gc] && section[:gc_table] } + sections ||= [{ table: table, format: format, failures: bench_failures, include_gc: include_gc, gc_tables: gc_tables }] + has_gc_summary = sections.any? { |section| section[:include_gc] && section[:gc_tables] } sections.each do |section| title = section[:title] output_str << "#{title}:\n" if title output_str << TableFormatter.new(section[:table], section[:format], section.fetch(:failures, {})).to_s + "\n" - if section[:include_gc] && section[:gc_table] && section[:gc_format] - output_str << (title ? "GC summary (#{title}):\n" : "GC summary:\n") - output_str << TableFormatter.new(section[:gc_table], section[:gc_format], {}).to_s + "\n" + next unless section[:include_gc] && section[:gc_tables] + + section[:gc_tables].each do |gc_table| + qualifiers = [gc_table[:scope], title].compact + output_str << gc_table[:name] + output_str << " (#{qualifiers.join(', ')})" unless qualifiers.empty? + output_str << ":\n" + output_str << TableFormatter.new(gc_table[:rows], Array.new(gc_table[:rows].first.size, "%s"), {}).to_s + output_str << "Hidden columns (zero or N/A in every row): #{gc_table[:hidden].join(', ')}\n" unless gc_table[:hidden].empty? + output_str << "\n" end end @@ -81,30 +88,30 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_ end end if has_gc_summary - output_str << "- GC summary compares #{base_name} → comparison. Ratio columns are #{base_name}/comparison; above 1 means the comparison spent less GC time.\n" - output_str << "- gc/iter, mark/iter, and sweep/iter ratio compare total GC (or phase) time per benchmark iteration, so they include both per-GC cost and GC frequency changes.\n" - output_str << "- gc/GC, mark/GC, and sweep/GC ratio divide average GC (or phase) time by the same run's GCs/iter count; they are not complete per-cycle attribution of process-wide GC.\n" - output_str << "- GCs/iter, major/iter, minor/iter, controller compacts/iter*, and minor GC % show #{base_name} → comparison values, not ratios. Rows with no GC activity are omitted.\n" + output_str << "- GC time ratios are #{base_name}/comparison; above 1 means the comparison spent less GC time.\n" + output_str << "- gc/iter, mark/iter, and sweep/iter compare total GC (or phase) time per benchmark iteration, so they include both per-GC cost and GC frequency changes.\n" + output_str << "- gc/GC, mark/GC, and sweep/GC divide average GC (or phase) time by the same run's GCs/iter count; they are not complete per-cycle attribution of process-wide GC.\n" + output_str << "- GC counts show #{base_name} → comparison values, not ratios. Rows with no GC activity are omitted.\n" end if include_pvalue output_str << "- ***: p < 0.001, **: p < 0.01, *: p < 0.05 (Welch's t-test)\n" end end - gc_headers = sections.filter_map { |section| section[:gc_table]&.first }.flatten - if gc_headers.include?('controller compacts/iter*') + gc_headers = sections.flat_map { |section| Array(section[:gc_tables]).flat_map { |gc_table| gc_table[:rows].first } } + if gc_headers.include?('compacts*') output_str << "GC metric notes:\n" - output_str << "- controller compacts/iter*: the main Ractor's GC.stat(:compact_count) delta per iteration. Every global compacting cycle increments compact_count in every object space. Do not sum it across workers.#{other_names.empty? ? '' : " Comparison tables show #{base_name} → comparison values."}\n" + output_str << "- compacts*: the main Ractor's GC.stat(:compact_count) delta per iteration. Every global compacting cycle increments compact_count in every object space. Do not sum it across workers.#{other_names.empty? ? '' : " Comparison tables show #{base_name} → comparison values."}\n" end - ractor_gc_sections = sections.select { |section| section[:gc_scope] == 'ractor-local-workload' && section[:gc_table] } + ractor_gc_sections = sections.select { |section| section[:gc_scope] == 'ractor-local-workload' && section[:gc_tables] } unless ractor_gc_sections.empty? modes = ractor_gc_sections.flat_map { |section| section.fetch(:ractor_gc_modes, []) } worker_mode = modes.include?(:worker) output_str << "Ractor GC scope note:\n" - scope_columns = +"- (worker sum) columns add Ractor-local counters across the sampled workers of each iteration" + scope_columns = +"- Tables marked worker sum add Ractor-local counters and GC times across the sampled workers of each iteration" scope_columns << (worker_mode ? "; the main Ractor performs the count-0 workload of per-worker benchmarks." : ".") scope_columns << " GC ms/worker divides each iteration's worker-sum GC time by its sampled worker count, then averages." if other_names.empty? - output_str << "#{scope_columns} Controller snapshots and per-worker heap detail are in the JSON output, not this table.\n" + output_str << "#{scope_columns} Controller snapshots and per-worker heap detail are in the JSON output, not these tables.\n" output_str << "- Ruby's Ractor-retirement GC (after a worker's stack is torn down) is not sampled." output_str << " Ractors created by the workload of a per-worker benchmark are not sampled." if worker_mode output_str << "\n" diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index efbcaad9..3683520b 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -117,7 +117,7 @@ def run include_pvalue: args.pvalue, zjit_stats: args.zjit_stats ) - table, format, gc_table, gc_format = builder.build + table, format, gc_tables = builder.build output_path = BenchmarkRunner.output_path(args.out_path, out_override: args.out_override) @@ -131,7 +131,7 @@ def run # Save the output in a text file that we can easily refer to output_sections = build_output_sections(ruby_descriptions.keys, bench_data, bench_harnesses, bench_failures) - output_str = BenchmarkRunner.build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: args.rss, include_gc: builder.include_gc?, include_pvalue: args.pvalue, gc_table: gc_table, gc_format: gc_format, sections: output_sections) + output_str = BenchmarkRunner.build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: args.rss, include_gc: builder.include_gc?, include_pvalue: args.pvalue, gc_tables: gc_tables, sections: output_sections) out_txt_path = output_path + ".txt" File.open(out_txt_path, "w") { |f| f.write output_str } @@ -193,7 +193,7 @@ def build_output_section(executable_names, bench_data, bench_failures, harness, zjit_stats: args.zjit_stats, row_layout: layout ) - table, format, gc_table, gc_format = builder.build + table, format, gc_tables = builder.build section = { title: harness, @@ -201,8 +201,7 @@ def build_output_section(executable_names, bench_data, bench_failures, harness, format: format, failures: slice_failures(bench_failures, bench_names), include_gc: builder.include_gc?, - gc_table: gc_table, - gc_format: gc_format, + gc_tables: gc_tables, } if ResultsTableBuilder.ractor_gc_data?(section_data) section[:gc_scope] = 'ractor-local-workload' diff --git a/lib/results_table_builder.rb b/lib/results_table_builder.rb index 8450331d..00e031fb 100644 --- a/lib/results_table_builder.rb +++ b/lib/results_table_builder.rb @@ -42,9 +42,7 @@ def build table << (entry.label_cells + build_stat_cells(entry.data_key)) end - gc_table = build_gc_summary_table - - [table, format, gc_table, build_gc_summary_format(gc_table)] + [table, format, build_gc_tables] end private @@ -122,26 +120,18 @@ def build_format gc_controller_compact_count_bench ].freeze - def build_gc_summary_table + def build_gc_tables return nil unless @include_gc - label_columns = ["bench", *@row_layout.extra_header_columns] - - if @other_names.empty? - return build_gc_absolute_table(label_columns) - end - - count_suffix = ractor_gc_table? ? " (worker sum)" : "" - - header = label_columns + (include_gc_comparison_name? ? ["comparison"] : []) - header += ["gc/iter ratio", "gc/GC ratio"] if include_gc_total_time? - header += ["mark/iter ratio", "sweep/iter ratio", "mark/GC ratio", "sweep/GC ratio"] - header << "global/iter ratio" if gc_series_present?('gc_global_count_bench') - header << "GCs/iter#{count_suffix}" << "major/iter#{count_suffix}" << "minor/iter#{count_suffix}" - header << "controller compacts/iter*" if gc_series_present?('gc_controller_compact_count_bench') - header << "minor GC %" + label_header = ["bench", *@row_layout.extra_header_columns] + tables = @other_names.empty? ? [build_gc_absolute_table(label_header)] : build_gc_comparison_tables(label_header) + tables.compact! + tables.empty? ? nil : tables + end - rows = [header] + def build_gc_comparison_tables(label_header) + label_header += ["comparison"] if include_gc_comparison_name? + rows = [] gc_entries.each do |entry| series_by_exe = @executable_names.map do |name| data = bench_data_for(name, entry.data_key) @@ -161,44 +151,118 @@ def build_gc_summary_table others.each_with_index do |other, i| next unless gc_activity?(*base.values, *other.values) - rows << gc_summary_row(gc_label_cells(entry), @other_names[i], base, other) + labels = gc_label_cells(entry) + labels << @other_names[i] if include_gc_comparison_name? + rows << [labels, [base, other]] end end - rows.size == 1 ? nil : rows + [ + assemble_gc_table("GC time ratios", label_header, rows, gc_ratio_columns), + assemble_gc_table("GC counts", label_header, rows, gc_count_columns), + ] end - def build_gc_absolute_table(label_columns) - ractor = ractor_gc_table? - count_suffix = ractor ? " (worker sum)" : "" - - header = label_columns + ["GC ms/iter#{count_suffix}"] - header << "GC ms/worker" if ractor - header += ["mark ms/iter#{count_suffix}", "sweep ms/iter#{count_suffix}", "GCs/iter#{count_suffix}", "major/iter#{count_suffix}", "minor/iter#{count_suffix}"] - header << "global/iter#{count_suffix}" if gc_series_present?('gc_global_count_bench') - header << "controller compacts/iter*" if gc_series_present?('gc_controller_compact_count_bench') + def gc_ratio_columns + columns = [] + if include_gc_total_time? + columns << ["gc/iter", ->(base, other) { ratio_cell(gc_ratio(base[:total], other[:total])) }] + columns << ["gc/GC", ->(base, other) { ratio_cell(per_gc_ratio(base, other, :total)) }] + end + columns << ["mark/iter", ->(base, other) { ratio_cell(gc_ratio(base[:mark], other[:mark])) }] + columns << ["sweep/iter", ->(base, other) { ratio_cell(gc_ratio(base[:sweep], other[:sweep])) }] + columns << ["mark/GC", ->(base, other) { ratio_cell(per_gc_ratio(base, other, :mark)) }] + columns << ["sweep/GC", ->(base, other) { ratio_cell(per_gc_ratio(base, other, :sweep)) }] + columns + end - rows = [header] + def gc_count_columns + columns = [ + ["GCs/iter", ->(base, other) { count_cell(base[:count], other[:count]) }], + ["major/iter", ->(base, other) { count_cell(base[:major], other[:major]) }], + ["minor/iter", ->(base, other) { count_cell(base[:minor], other[:minor]) }], + ] + if gc_series_present?('gc_global_count_bench') + columns << ["global/iter", ->(base, other) { count_cell(base[:global], other[:global]) }] + end + if gc_series_present?('gc_controller_compact_count_bench') + columns << ["compacts*", ->(base, other) { count_cell(base[:compact], other[:compact]) }] + end + columns << ["minor GC %", ->(base, other) { minor_percent_cell(base, other) }] + columns + end - gc_entries.each do |entry| + def build_gc_absolute_table(label_header) + rows = gc_entries.filter_map do |entry| data = bench_data_for(@base_name, entry.data_key) - next unless GC_SERIES_KEYS.any? { |key| data.key?(key) } + [gc_label_cells(entry), [data]] if GC_SERIES_KEYS.any? { |key| data.key?(key) } + end + assemble_gc_table("GC summary", label_header, rows, gc_absolute_columns) + end - cells = [format_gc_series_mean_precise(data['gc_total_time_bench'])] - cells << gc_ms_per_worker_cell(data['gc_total_time_bench'], data['gc_worker_samples']) if ractor - cells += [ - format_gc_series_mean_precise(data['gc_marking_time_bench']), - format_gc_series_mean_precise(data['gc_sweeping_time_bench']), - format_gc_series_mean(gc_count_series(data)), - format_gc_series_mean(data['gc_major_count_bench']), - format_gc_series_mean(data['gc_minor_count_bench']), - ] - cells << format_gc_series_mean(data['gc_global_count_bench']) if gc_series_present?('gc_global_count_bench') - cells << format_gc_series_mean(data['gc_controller_compact_count_bench']) if gc_series_present?('gc_controller_compact_count_bench') - rows << gc_label_cells(entry) + cells + def gc_absolute_columns + columns = [["GC ms/iter", ->(data) { mean_cell(data['gc_total_time_bench'], precise: true) }]] + columns << ["GC ms/worker", ->(data) { ms_per_worker_cell(data) }] if ractor_gc_table? + columns += [ + ["mark ms/iter", ->(data) { mean_cell(data['gc_marking_time_bench'], precise: true) }], + ["sweep ms/iter", ->(data) { mean_cell(data['gc_sweeping_time_bench'], precise: true) }], + ["GCs/iter", ->(data) { mean_cell(gc_count_series(data)) }], + ["major/iter", ->(data) { mean_cell(data['gc_major_count_bench']) }], + ["minor/iter", ->(data) { mean_cell(data['gc_minor_count_bench']) }], + ] + if gc_series_present?('gc_global_count_bench') + columns << ["global/iter", ->(data) { mean_cell(data['gc_global_count_bench']) }] + end + if gc_series_present?('gc_controller_compact_count_bench') + columns << ["compacts*", ->(data) { mean_cell(data['gc_controller_compact_count_bench']) }] end + columns + end + + def assemble_gc_table(name, label_header, rows, columns) + return nil if rows.empty? - rows.size == 1 ? nil : rows + cells = rows.map { |(_labels, args)| columns.map { |(_header, cell)| cell.call(*args) } } + shown = columns.each_index.select { |i| cells.any? { |row_cells| row_cells[i][1] } } + + body = rows.each_with_index.map { |(labels, _args), r| labels + shown.map { |i| cells[r][i][0] } } + { + name: name, + scope: ractor_gc_table? ? "worker sum" : nil, + rows: [label_header + shown.map { |i| columns[i][0] }] + body, + hidden: (columns.each_index.to_a - shown).map { |i| columns[i][0] }, + } + end + + def ratio_cell(text) + [text, text != "N/A"] + end + + def count_cell(base, other) + [gc_count_cell(base, other), mean_positive?(base) || mean_positive?(other)] + end + + def minor_percent_cell(base, other) + data = [gc_minor_percent(base[:minor], base[:count]), gc_minor_percent(other[:minor], other[:count])].any? { |pct| pct&.positive? } + [gc_minor_percent_cell(base, other), data] + end + + def mean_cell(values, precise: false) + text = precise ? format_gc_series_mean_precise(values) : format_gc_series_mean(values) + [text, mean_positive?(values)] + end + + def ms_per_worker_cell(data) + text = gc_ms_per_worker_cell(data['gc_total_time_bench'], data['gc_worker_samples']) + [text, text != "N/A" && mean_positive?(data['gc_total_time_bench'])] + end + + def mean_positive?(values) + numeric_series?(values) && mean(values) > 0.0 + end + + def per_gc_ratio(base, other, key) + scalar_ratio(gc_time_per_gc(base[key], base[:count]), gc_time_per_gc(other[key], other[:count])) end def gc_series_present?(key) @@ -233,12 +297,6 @@ def include_gc_total_time? end end - def build_gc_summary_format(gc_table) - return nil unless gc_table - - Array.new(gc_table.first.size, "%s") - end - def build_stat_cells(bench_name) t0s = extract_first_iteration_times(bench_name) times_no_warmup = extract_benchmark_times(bench_name) @@ -316,26 +374,6 @@ def include_gc_comparison_name? @other_names.size > 1 end - def gc_summary_row(label_cells, name, base, other) - row = label_cells - row << name if include_gc_comparison_name? - if include_gc_total_time? - row << gc_ratio(base[:total], other[:total]) - row << scalar_ratio(gc_time_per_gc(base[:total], base[:count]), gc_time_per_gc(other[:total], other[:count])) - end - row << gc_ratio(base[:mark], other[:mark]) - row << gc_ratio(base[:sweep], other[:sweep]) - row << scalar_ratio(gc_time_per_gc(base[:mark], base[:count]), gc_time_per_gc(other[:mark], other[:count])) - row << scalar_ratio(gc_time_per_gc(base[:sweep], base[:count]), gc_time_per_gc(other[:sweep], other[:count])) - row << gc_ratio(base[:global], other[:global]) if gc_series_present?('gc_global_count_bench') - row << gc_count_cell(base[:count], other[:count]) - row << gc_count_cell(base[:major], other[:major]) - row << gc_count_cell(base[:minor], other[:minor]) - row << gc_count_cell(base[:compact], other[:compact]) if gc_series_present?('gc_controller_compact_count_bench') - row << gc_minor_percent_cell(base, other) - row - end - def numeric_series?(values) values.is_a?(Array) && !values.empty? && values.all? { |v| v.is_a?(Numeric) } end diff --git a/test/benchmark_runner_cli_test.rb b/test/benchmark_runner_cli_test.rb index 5e19a50b..1af29335 100644 --- a/test/benchmark_runner_cli_test.rb +++ b/test/benchmark_runner_cli_test.rb @@ -160,12 +160,14 @@ def create_args(overrides = {}) assert_equal ['object-new', '0'], ractor[:table][1][0..1] assert_equal ['', '2'], ractor[:table][2][0..1] - assert_equal ['object-new', '0'], ractor[:gc_table][1][0..1] - assert_equal ['object-new', '2'], ractor[:gc_table][2][0..1] - assert_equal ['bench', 'ractors', 'GC ms/iter (worker sum)', 'GC ms/worker', 'mark ms/iter (worker sum)', 'sweep ms/iter (worker sum)', 'GCs/iter (worker sum)', 'major/iter (worker sum)', 'minor/iter (worker sum)'], ractor[:gc_table][0] - ractor[:gc_table].flatten.each { |cell| refute_includes cell.to_s, "\x00" } - - assert_nil normal[:gc_table] + gc_table = ractor[:gc_tables].first + assert_equal 'worker sum', gc_table[:scope] + assert_equal ['object-new', '0'], gc_table[:rows][1][0..1] + assert_equal ['object-new', '2'], gc_table[:rows][2][0..1] + assert_equal ['bench', 'ractors', 'GC ms/iter', 'GC ms/worker', 'mark ms/iter', 'sweep ms/iter', 'GCs/iter', 'major/iter', 'minor/iter'], gc_table[:rows][0] + gc_table[:rows].flatten.each { |cell| refute_includes cell.to_s, "\x00" } + + assert_nil normal[:gc_tables] output = BenchmarkRunner.build_output_text({ 'ruby' => 'ruby 4.1.0dev' }, nil, nil, {}, sections: sections) assert_match(/Ractor GC scope note:/, output) diff --git a/test/benchmark_runner_test.rb b/test/benchmark_runner_test.rb index 7792d8b5..7c16d854 100644 --- a/test/benchmark_runner_test.rb +++ b/test/benchmark_runner_test.rb @@ -409,7 +409,7 @@ assert_includes result, "- ***: p < 0.001, **: p < 0.01, *: p < 0.05 (Welch's t-test)" end - it 'prints compact GC comparison table and legend when include_gc is true' do + it 'prints the GC ratio and count tables and their legend when include_gc is true' do ruby_descriptions = { 'ruby-base' => 'ruby 3.3.0', 'ruby-exp' => 'ruby 3.3.0 experiment' @@ -419,22 +419,57 @@ ['fib', '100.0', '50.0', '2.000'] ] format = ['%s', '%s', '%s', '%s'] - gc_table = [ - ['bench', 'mark/iter ratio', 'sweep/iter ratio', 'mark/GC ratio', 'sweep/GC ratio', 'major/iter', 'minor/iter', 'minor GC %'], - ['fib', '2.000', '1.250', '1.000', '0.625', ' 2.0 → 1.0', ' 8.0 → 4.0', ' 80% → 80%'] + gc_tables = [ + { name: 'GC time ratios', scope: nil, hidden: [], rows: [ + ['bench', 'mark/iter', 'sweep/iter', 'mark/GC', 'sweep/GC'], + ['fib', '2.000', '1.250', '1.000', '0.625'] + ] }, + { name: 'GC counts', scope: nil, hidden: [], rows: [ + ['bench', 'major/iter', 'minor/iter', 'minor GC %'], + ['fib', ' 2.0 → 1.0', ' 8.0 → 4.0', ' 80% → 80%'] + ] } ] - gc_format = ['%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s'] - bench_failures = {} result = BenchmarkRunner.build_output_text( - ruby_descriptions, table, format, bench_failures, include_gc: true, gc_table: gc_table, gc_format: gc_format + ruby_descriptions, table, format, {}, include_gc: true, gc_tables: gc_tables ) - assert_includes result, "GC summary:\n" - assert_includes result, 'mark/GC ratio' + assert_includes result, "GC time ratios:\n" + assert_includes result, "GC counts:\n" + assert_operator result.index("GC time ratios:\n"), :<, result.index("GC counts:\n") + assert_includes result, 'mark/GC' assert_includes result, ' 2.0 → 1.0' - assert_includes result, '- GC summary compares ruby-base → comparison. Ratio columns are ruby-base/comparison; above 1 means the comparison spent less GC time.' - refute_includes result, 'mark ruby-base/ruby-exp:' + assert_includes result, '- GC time ratios are ruby-base/comparison; above 1 means the comparison spent less GC time.' + refute_includes result, 'Hidden columns' + end + + it 'titles a worker-sum GC table with its scope and the section title, and lists hidden columns' do + ruby_descriptions = { 'ruby' => 'ruby 4.1.0dev' } + sections = [ + { + title: 'harness-ractor', + table: [['bench', 'ractors', 'ruby (ms)'], ['object-new', '1', '200.0']], + format: ['%s', '%s', '%.1f'], + failures: {}, + include_gc: true, + gc_tables: [ + { name: 'GC summary', scope: 'worker sum', hidden: ['global/iter', 'compacts*'], rows: [ + ['bench', 'ractors', 'GC ms/iter'], ['object-new', '1', '4.000'] + ] } + ], + gc_scope: 'ractor-local-workload', + ractor_gc_modes: [:worker], + } + ] + + result = BenchmarkRunner.build_output_text( + ruby_descriptions, sections.first[:table], sections.first[:format], {}, sections: sections + ) + + assert_includes result, "GC summary (worker sum, harness-ractor):\n" + assert_includes result, "Hidden columns (zero or N/A in every row): global/iter, compacts*\n" + refute_includes result, 'GC metric notes:', 'a hidden compaction column needs no note' + assert_includes result, '- Tables marked worker sum add Ractor-local counters' end it 'explains the controller compaction column when a GC table shows it' do @@ -447,18 +482,19 @@ ['fib', '100.0', '50.0', '2.000'] ] format = ['%s', '%s', '%s', '%s'] - gc_table = [ - ['bench', 'gc/iter ratio', 'global/iter ratio', 'GCs/iter', 'controller compacts/iter*'], - ['fib', '2.000', '1.500', '10.0 → 5.0', ' 1.0 → 2.0'] + gc_tables = [ + { name: 'GC counts', scope: nil, hidden: [], rows: [ + ['bench', 'GCs/iter', 'compacts*'], + ['fib', '10.0 → 5.0', ' 1.0 → 2.0'] + ] } ] - gc_format = ['%s', '%s', '%s', '%s', '%s'] result = BenchmarkRunner.build_output_text( - ruby_descriptions, table, format, {}, include_gc: true, gc_table: gc_table, gc_format: gc_format + ruby_descriptions, table, format, {}, include_gc: true, gc_tables: gc_tables ) assert_includes result, "GC metric notes:\n" - assert_includes result, "- controller compacts/iter*: the main Ractor's GC.stat(:compact_count) delta per iteration." + assert_includes result, "- compacts*: the main Ractor's GC.stat(:compact_count) delta per iteration." assert_includes result, 'Every global compacting cycle increments compact_count in every object space.' assert_includes result, 'Do not sum it across workers.' assert_includes result, 'Comparison tables show ruby-base → comparison values.' @@ -469,19 +505,20 @@ ruby_descriptions = { 'ruby' => 'ruby 4.1.0dev' } table = [['bench', 'ruby (ms)'], ['fib', '100.0']] format = ['%s', '%s'] - gc_table = [ - ['bench', 'GC ms/iter', 'GCs/iter', 'global/iter', 'controller compacts/iter*'], - ['fib', '4.000', '10.0', '1.0', '0.0'] + gc_tables = [ + { name: 'GC summary', scope: nil, hidden: [], rows: [ + ['bench', 'GC ms/iter', 'GCs/iter', 'global/iter', 'compacts*'], + ['fib', '4.000', '10.0', '1.0', '0.0'] + ] } ] - gc_format = ['%s', '%s', '%s', '%s', '%s'] result = BenchmarkRunner.build_output_text( - ruby_descriptions, table, format, {}, include_gc: true, gc_table: gc_table, gc_format: gc_format + ruby_descriptions, table, format, {}, include_gc: true, gc_tables: gc_tables ) refute_includes result, 'Legend:' assert_includes result, "GC metric notes:\n" - assert_includes result, "- controller compacts/iter*: the main Ractor's GC.stat(:compact_count) delta per iteration." + assert_includes result, "- compacts*: the main Ractor's GC.stat(:compact_count) delta per iteration." refute_includes result, 'Comparison tables show' refute_includes result, 'global GCs/iter' end @@ -496,19 +533,20 @@ ['fib', '100.0', '50.0', '2.000'] ] format = ['%s', '%s', '%s', '%s'] - gc_table = [ - ['bench', 'mark/iter ratio', 'GCs/iter', 'global/iter'], - ['fib', '2.000', '10.0 → 5.0', ' 2.0 → 1.0'] + gc_tables = [ + { name: 'GC counts', scope: nil, hidden: [], rows: [ + ['bench', 'GCs/iter', 'major/iter'], + ['fib', '10.0 → 5.0', ' 2.0 → 1.0'] + ] } ] - gc_format = ['%s', '%s', '%s', '%s'] result = BenchmarkRunner.build_output_text( - ruby_descriptions, table, format, {}, include_gc: true, gc_table: gc_table, gc_format: gc_format + ruby_descriptions, table, format, {}, include_gc: true, gc_tables: gc_tables ) refute_includes result, 'GC metric notes:' assert_includes result, "the same run's GCs/iter count" - assert_includes result, 'show ruby-base → comparison values, not ratios' + assert_includes result, 'GC counts show ruby-base → comparison values, not ratios' end it 'omits the Ractor scope note when the section rendered no GC table' do @@ -520,7 +558,7 @@ format: ['%s', '%s', '%.1f'], failures: {}, include_gc: true, - gc_table: nil, + gc_tables: nil, gc_scope: 'ractor-local-workload', } ] @@ -629,8 +667,7 @@ format: ['%s', '%.1f', '%.1f'], failures: {}, include_gc: true, - gc_table: [['bench', 'mark/iter ratio'], ['gcbench', '1.100']], - gc_format: ['%s', '%s'], + gc_tables: [{ name: 'GC time ratios', scope: nil, hidden: [], rows: [['bench', 'mark/iter'], ['gcbench', '1.100']] }], } ] @@ -638,8 +675,8 @@ ruby_descriptions, sections.first[:table], sections.first[:format], {}, sections: sections ) - assert_includes result, "GC summary (harness-gc):\n" - assert_includes result, '- GC summary compares ruby → comparison.' + assert_includes result, "GC time ratios (harness-gc):\n" + assert_includes result, '- GC time ratios are ruby/comparison;' end it 'omits legend when no other executables' do diff --git a/test/results_table_builder_test.rb b/test/results_table_builder_test.rb index 97246775..af523ebe 100644 --- a/test/results_table_builder_test.rb +++ b/test/results_table_builder_test.rb @@ -599,7 +599,7 @@ end describe 'GC summary data' do - it 'keeps GC columns out of the main table and builds a compact GC comparison table' do + it 'keeps GC columns out of the main table and splits the comparison into ratio and count tables' do bench_data = { 'ruby-base' => { 'fib' => { @@ -630,18 +630,23 @@ bench_data: bench_data ) - table, format, gc_table, gc_format = builder.build + table, format, gc_tables = builder.build assert_equal ['bench', 'ruby-base (ms)', 'ruby-exp (ms)', 'ruby-exp 1st itr', 'ruby-base/ruby-exp'], table[0] assert_equal ['%s', '%s', '%s', '%.3f', '%s'], format + ratios, counts = gc_tables + assert_equal ['GC time ratios', 'GC counts'], gc_tables.map { |t| t[:name] } + assert_equal [nil, nil], gc_tables.map { |t| t[:scope] }, 'non-Ractor tables are not worker sums' assert_equal [ - 'bench', 'mark/iter ratio', 'sweep/iter ratio', 'mark/GC ratio', 'sweep/GC ratio', 'GCs/iter', 'major/iter', 'minor/iter', 'minor GC %' - ], gc_table[0] - assert_equal ['%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s'], gc_format + ['bench', 'mark/iter', 'sweep/iter', 'mark/GC', 'sweep/GC'], + ['fib', '1.333', '1.000', '0.667', '0.500'] + ], ratios[:rows] assert_equal [ - 'fib', '1.333', '1.000', '0.667', '0.500', '10.0 → 5.0', ' 2.0 → 1.0', ' 8.0 → 4.0', ' 80% → 80%' - ], gc_table[1] + ['bench', 'GCs/iter', 'major/iter', 'minor/iter', 'minor GC %'], + ['fib', '10.0 → 5.0', ' 2.0 → 1.0', ' 8.0 → 4.0', ' 80% → 80%'] + ], counts[:rows] + assert_equal [[], []], gc_tables.map { |t| t[:hidden] } end it 'omits benchmarks with no GC activity from the GC summary' do @@ -675,13 +680,12 @@ bench_data: bench_data ) - _table, _format, gc_table, gc_format = builder.build + _table, _format, gc_tables = builder.build - assert_nil gc_table - assert_nil gc_format + assert_nil gc_tables end - it 'builds a non-Ractor absolute GC table without worker-sum suffixes or GC ms/worker' do + it 'builds a non-Ractor absolute GC table and hides a column that is zero in every row' do bench_data = { 'ruby' => { 'gcbench' => { @@ -705,14 +709,69 @@ bench_data: bench_data ) - _table, _format, gc_table, gc_format = builder.build + _table, _format, gc_tables = builder.build + assert_equal 1, gc_tables.size + gc_table = gc_tables.first + assert_equal 'GC summary', gc_table[:name] + assert_nil gc_table[:scope] assert_equal [ - 'bench', 'GC ms/iter', 'mark ms/iter', 'sweep ms/iter', 'GCs/iter', 'major/iter', 'minor/iter', - 'global/iter', 'controller compacts/iter*' - ], gc_table[0] - assert_equal ['%s'] * 9, gc_format - assert_equal ['gcbench', '5.000', '2.000', '1.000', '11.0', '2.0', '8.0', '1.0', '0.0'], gc_table[1] + ['bench', 'GC ms/iter', 'mark ms/iter', 'sweep ms/iter', 'GCs/iter', 'major/iter', 'minor/iter', 'global/iter'], + ['gcbench', '5.000', '2.000', '1.000', '11.0', '2.0', '8.0', '1.0'] + ], gc_table[:rows] + assert_equal ['compacts*'], gc_table[:hidden] + end + + it 'hides minor/iter and minor GC % when every comparison row ran only major GCs' do + blob = ->(major) do + { 'warmup' => [], 'bench' => [0.1], 'rss' => 10, 'gc_major_count_bench' => [major], 'gc_minor_count_bench' => [0] } + end + builder = ResultsTableBuilder.new( + executable_names: ['ruby-base', 'ruby-exp'], + bench_data: { 'ruby-base' => { 'fib' => blob.call(3) }, 'ruby-exp' => { 'fib' => blob.call(2) } } + ) + + _table, _format, gc_tables = builder.build + + ratios, counts = gc_tables + assert_equal [['bench', 'GCs/iter', 'major/iter'], ['fib', ' 3.0 → 2.0', ' 3.0 → 2.0']], counts[:rows] + assert_equal ['minor/iter', 'minor GC %'], counts[:hidden], 'a 0% → 0% column has no data' + assert_equal 'GC time ratios', ratios[:name], 'a table with rows stays even when every column is hidden' + assert_equal [['bench'], ['fib']], ratios[:rows] + assert_equal ['mark/iter', 'sweep/iter', 'mark/GC', 'sweep/GC'], ratios[:hidden] + end + + it 'keeps a single-executable GC table whose GC series are zero in every row' do + blob = { + 'warmup' => [], 'bench' => [0.1], 'rss' => 10, + 'gc_total_time_bench' => [0.0], 'gc_count_bench' => [0], 'gc_major_count_bench' => [0], 'gc_minor_count_bench' => [0] + } + builder = ResultsTableBuilder.new(executable_names: ['ruby'], bench_data: { 'ruby' => { 'fib' => blob } }) + + _table, _format, gc_tables = builder.build + + refute_nil gc_tables, 'sampled GC data with no activity still produces a table' + assert_equal [['bench'], ['fib']], gc_tables.first[:rows] + assert_equal ['GC ms/iter', 'mark ms/iter', 'sweep ms/iter', 'GCs/iter', 'major/iter', 'minor/iter'], gc_tables.first[:hidden] + end + + it 'hides a count column whose partially-null series render as N/A on both sides' do + blob = ->(global) do + { + 'warmup' => [], 'bench' => [0.1, 0.1], 'rss' => 10, + 'gc_major_count_bench' => [1, 1], 'gc_minor_count_bench' => [2, 2], 'gc_global_count_bench' => global + } + end + builder = ResultsTableBuilder.new( + executable_names: ['ruby-base', 'ruby-exp'], + bench_data: { 'ruby-base' => { 'fib' => blob.call([nil, 2]) }, 'ruby-exp' => { 'fib' => blob.call([nil, 3]) } } + ) + + _table, _format, gc_tables = builder.build + + counts = gc_tables.find { |t| t[:name] == 'GC counts' } + refute_includes counts[:rows][0], 'global/iter', 'a column of N/A → N/A cells has no data' + assert_includes counts[:hidden], 'global/iter' end end @@ -890,24 +949,24 @@ def build_ractor_gc(bench_data) } } - table, _format, gc_table, gc_format = build_ractor_gc(bench_data) + table, _format, gc_tables = build_ractor_gc(bench_data) assert_equal ['bench', 'ractors', 'base (ms)', 'candidate (ms)', 'candidate 1st itr', 'base/candidate'], table[0] - assert_equal [ - 'bench', 'ractors', 'gc/iter ratio', 'gc/GC ratio', 'mark/iter ratio', 'sweep/iter ratio', - 'mark/GC ratio', 'sweep/GC ratio', 'GCs/iter (worker sum)', 'major/iter (worker sum)', 'minor/iter (worker sum)', 'minor GC %' - ], gc_table[0] - assert_equal ['%s'] * gc_table[0].size, gc_format - - rows = gc_table[1..].to_h { |row| [row[1], row] } - assert_equal %w[0 2], gc_table[1..].map { |row| row[1] } + assert_equal ['worker sum', 'worker sum'], gc_tables.map { |t| t[:scope] } + ratios, counts = gc_tables.map { |t| t[:rows] } + assert_equal ['bench', 'ractors', 'gc/iter', 'gc/GC', 'mark/iter', 'sweep/iter', 'mark/GC', 'sweep/GC'], ratios[0] + assert_equal ['bench', 'ractors', 'GCs/iter', 'major/iter', 'minor/iter', 'minor GC %'], counts[0] - assert_equal ['object-new', 'object-new'], gc_table[1..].map(&:first) + assert_equal %w[0 2], ratios[1..].map { |row| row[1] } + assert_equal %w[0 2], counts[1..].map { |row| row[1] }, 'both tables list the same rows' + assert_equal ['object-new', 'object-new'], ratios[1..].map(&:first) - assert_equal ['object-new', '0', '2.000', '1.000', '1.000', '1.000', '0.500', '0.500', ' 4.0 → 2.0', ' 1.0 → 1.0', ' 3.0 → 1.0', ' 75% → 50%'], rows['0'] - assert_equal ['object-new', '2', '4.000', '2.000', '2.000', '2.000', '1.000', '1.000', ' 8.0 → 4.0', ' 2.0 → 1.0', ' 6.0 → 3.0', ' 75% → 75%'], rows['2'] + assert_equal ['object-new', '0', '2.000', '1.000', '1.000', '1.000', '0.500', '0.500'], ratios[1] + assert_equal ['object-new', '2', '4.000', '2.000', '2.000', '2.000', '1.000', '1.000'], ratios[2] + assert_equal ['object-new', '0', ' 4.0 → 2.0', ' 1.0 → 1.0', ' 3.0 → 1.0', ' 75% → 50%'], counts[1] + assert_equal ['object-new', '2', ' 8.0 → 4.0', ' 2.0 → 1.0', ' 6.0 → 3.0', ' 75% → 75%'], counts[2] - gc_table.flatten.each { |cell| refute_includes cell.to_s, "\x00" } + (ratios + counts).flatten.each { |cell| refute_includes cell.to_s, "\x00" } end it 'renders N/A for a count missing optional GC data instead of reusing another count' do @@ -926,9 +985,9 @@ def build_ractor_gc(bench_data) } } - _table, _format, gc_table, _gc_format = build_ractor_gc(bench_data) + _table, _format, gc_tables = build_ractor_gc(bench_data) - rows = gc_table[1..].to_h { |row| [row[1], row] } + rows = gc_tables.first[:rows][1..].to_h { |row| [row[1], row] } assert_equal '2.000', rows['0'][2], 'count 0 keeps its own gc/iter ratio' assert_equal '1.000', rows['0'][4] assert_equal 'N/A', rows['2'][2], 'missing total-time series must not reuse count 0 data or zero' @@ -947,12 +1006,13 @@ def build_ractor_gc(bench_data) } } - _table, _format, gc_table, gc_format = build_ractor_gc(bench_data) + _table, _format, gc_tables = build_ractor_gc(bench_data) - assert_equal ['bench', 'ractors', 'GC ms/iter (worker sum)', 'GC ms/worker', 'mark ms/iter (worker sum)', 'sweep ms/iter (worker sum)', 'GCs/iter (worker sum)', 'major/iter (worker sum)', 'minor/iter (worker sum)'], gc_table[0] - assert_equal ['%s'] * 9, gc_format - assert_equal ['object-new', '0', '0.000', '0.000', '0.000', 'N/A', '0.0', '0.0', '0.0'], gc_table[1] - assert_equal ['object-new', '2', '4.000', '4.000', '2.000', '1.000', '5.0', '1.0', '4.0'], gc_table[2] + gc_table = gc_tables.first + assert_equal 'worker sum', gc_table[:scope] + assert_equal ['bench', 'ractors', 'GC ms/iter', 'GC ms/worker', 'mark ms/iter', 'sweep ms/iter', 'GCs/iter', 'major/iter', 'minor/iter'], gc_table[:rows][0] + assert_equal ['object-new', '0', '0.000', '0.000', '0.000', 'N/A', '0.0', '0.0', '0.0'], gc_table[:rows][1] + assert_equal ['object-new', '2', '4.000', '4.000', '2.000', '1.000', '5.0', '1.0', '4.0'], gc_table[:rows][2] end it 'detects GC data from any recognized series, not just marking time' do @@ -964,10 +1024,12 @@ def build_ractor_gc(bench_data) } } - _table, _format, gc_table, _gc_format = build_ractor_gc(bench_data) + _table, _format, gc_tables = build_ractor_gc(bench_data) - refute_nil gc_table, 'a blob with only count series is still GC data' - assert_equal ['object-new', '0', 'N/A', 'N/A', 'N/A', 'N/A', '6.0', '2.0', '4.0'], gc_table[1] + refute_nil gc_tables, 'a blob with only count series is still GC data' + gc_table = gc_tables.first + assert_equal [['bench', 'ractors', 'GCs/iter', 'major/iter', 'minor/iter'], ['object-new', '0', '6.0', '2.0', '4.0']], gc_table[:rows] + assert_equal ['GC ms/iter', 'GC ms/worker', 'mark ms/iter', 'sweep ms/iter'], gc_table[:hidden], 'time columns without data in any row are hidden' end it 'derives GC ms/worker by dividing each iteration by its sampled worker count' do @@ -994,14 +1056,15 @@ def build_ractor_gc(bench_data) } } - _table, _format, gc_table, _gc_format = build_ractor_gc(bench_data) + _table, _format, gc_tables = build_ractor_gc(bench_data) + gc_table = gc_tables.first[:rows] header = gc_table[0] ms_per_worker_idx = header.index('GC ms/worker') refute_nil ms_per_worker_idx rows = gc_table[1..].to_h { |row| [row[0], row] } assert_equal '6.000', rows['obj-2w'][ms_per_worker_idx] - assert_equal '9.000', rows['obj-2w'][header.index('GC ms/iter (worker sum)')] + assert_equal '9.000', rows['obj-2w'][header.index('GC ms/iter')] assert_equal 'N/A', rows['obj-mismatch'][ms_per_worker_idx], 'mismatched series lengths' assert_equal 'N/A', rows['obj-empty'][ms_per_worker_idx], 'empty worker list' assert_equal 'N/A', rows['obj-nil-total'][ms_per_worker_idx], 'unavailable total' @@ -1020,13 +1083,14 @@ def build_ractor_gc(bench_data) 'candidate' => { 'object-new' => ractor_gc_blob('0' => { bench: [1.0, 1.0], gc: group.call(12.0, 6, 1, 2) }) } } - _table, _format, gc_table, _gc_format = build_ractor_gc(bench_data) + _table, _format, gc_tables = build_ractor_gc(bench_data) - header = gc_table[0] - row = gc_table[1] - assert_equal '2.500', row[header.index('gc/iter ratio')] - assert_equal '1.500', row[header.index('gc/GC ratio')] - assert_equal '10.0 → 6.0', row[header.index('GCs/iter (worker sum)')] + ratios, counts = gc_tables.map { |t| t[:rows] } + assert_equal '2.500', ratios[1][ratios[0].index('gc/iter')] + assert_equal '1.500', ratios[1][ratios[0].index('gc/GC')] + header = counts[0] + row = counts[1] + assert_equal '10.0 → 6.0', row[header.index('GCs/iter')] assert_equal ' 10% → 33%', row[header.index('minor GC %')], 'minor GC % divides by the direct count (1/10, 2/6), not major + minor' end @@ -1042,14 +1106,15 @@ def build_ractor_gc(bench_data) } } - _table, _format, gc_table, _gc_format = build_ractor_gc(bench_data) + _table, _format, gc_tables = build_ractor_gc(bench_data) + gc_table = gc_tables.first[:rows] header = gc_table[0] - gcs_idx = header.index('GCs/iter (worker sum)') + gcs_idx = header.index('GCs/iter') rows = gc_table[1..].to_h { |row| [row[0], row] } assert_equal '10.0', rows['legacy'][gcs_idx], 'absent direct-count key sums major and minor' assert_equal 'N/A', rows['null-count'][gcs_idx], 'present nulls stay unavailable instead of falling back' - assert_equal '2.0', rows['null-count'][header.index('major/iter (worker sum)')], 'neighboring values survive the null series' + assert_equal '2.0', rows['null-count'][header.index('major/iter')], 'neighboring values survive the null series' end it 'renders global GC and controller compaction columns in a Ractor comparison' do @@ -1065,23 +1130,18 @@ def build_ractor_gc(bench_data) 'candidate' => { 'object-new' => ractor_gc_blob('0' => { bench: [1.0, 1.0], gc: group.call(2, 3) }) } } - _table, _format, gc_table, _gc_format = build_ractor_gc(bench_data) + _table, _format, gc_tables = build_ractor_gc(bench_data) - header = gc_table[0] - assert_equal [ - 'bench', 'ractors', 'gc/iter ratio', 'gc/GC ratio', 'mark/iter ratio', 'sweep/iter ratio', - 'mark/GC ratio', 'sweep/GC ratio', 'global/iter ratio', - 'GCs/iter (worker sum)', 'major/iter (worker sum)', 'minor/iter (worker sum)', - 'controller compacts/iter*', 'minor GC %' - ], header - row = gc_table[1] - assert_equal '2.000', row[header.index('global/iter ratio')], 'base mean divided by comparison mean' - assert_equal ' 1.0 → 3.0', row[header.index('controller compacts/iter*')] - - assert_equal ' 38% → 50%', row[header.index('minor GC %')], 'the minor GC % denominator includes global cycles (3/8, not 3/4)' + ratios, counts = gc_tables.map { |t| t[:rows] } + assert_equal ['bench', 'ractors', 'gc/iter', 'gc/GC', 'mark/iter', 'sweep/iter', 'mark/GC', 'sweep/GC'], ratios[0], 'only time ratios' + assert_equal ['bench', 'ractors', 'GCs/iter', 'major/iter', 'minor/iter', 'global/iter', 'compacts*', 'minor GC %'], counts[0] + assert_equal ' 4.0 → 2.0', counts[1][counts[0].index('global/iter')], 'global GCs are counts, not a time ratio' + assert_equal ' 1.0 → 3.0', counts[1][counts[0].index('compacts*')] + + assert_equal ' 38% → 50%', counts[1][counts[0].index('minor GC %')], 'the minor GC % denominator includes global cycles (3/8, not 3/4)' end - it 'renders N/A for a partially-null series without losing the row or its neighbors' do + it 'hides a series that a null entry makes unavailable in every row, keeping the row and its neighbors' do partial_group = gc_group(total: [4.0, 4.0], major: [1, 1], minor: [3, 3], mark: [nil, 1.0], sweep: [1.0, 1.0]) full_group = gc_group(total: [4.0, 4.0], major: [1, 1], minor: [3, 3], mark: [2.0, 2.0], sweep: [1.0, 1.0]) bench_data = { @@ -1089,15 +1149,14 @@ def build_ractor_gc(bench_data) 'candidate' => { 'object-new' => ractor_gc_blob('0' => { bench: [1.0, 1.0], gc: full_group }) } } - _table, _format, gc_table, _gc_format = build_ractor_gc(bench_data) + _table, _format, gc_tables = build_ractor_gc(bench_data) - refute_nil gc_table, 'the nil entry must not hide a row whose other metrics have activity' - header = gc_table[0] - row = gc_table[1] - assert_equal 'N/A', row[header.index('mark/iter ratio')], 'a null entry makes the whole series unavailable' - assert_equal 'N/A', row[header.index('mark/GC ratio')] - assert_equal '1.000', row[header.index('sweep/iter ratio')], 'neighboring series keep their values' - assert_equal ' 4.0 → 4.0', row[header.index('GCs/iter (worker sum)')] + refute_nil gc_tables, 'the nil entry must not hide a row whose other metrics have activity' + ratios, counts = gc_tables + assert_equal ['mark/iter', 'mark/GC'], ratios[:hidden], 'a null entry makes the whole series unavailable in the only row' + header = ratios[:rows][0] + assert_equal '1.000', ratios[:rows][1][header.index('sweep/iter')], 'neighboring series keep their values' + assert_equal ' 4.0 → 4.0', counts[:rows][1][counts[:rows][0].index('GCs/iter')] end it 'shows global and compaction columns in a single-executable Ractor table' do @@ -1106,14 +1165,14 @@ def build_ractor_gc(bench_data) 'reference' => { 'object-new' => ractor_gc_blob('0' => { bench: [1.0, 1.0], gc: group }) } } - _table, _format, gc_table, _gc_format = build_ractor_gc(bench_data) + _table, _format, gc_tables = build_ractor_gc(bench_data) + gc_table = gc_tables.first assert_equal [ - 'bench', 'ractors', 'GC ms/iter (worker sum)', 'GC ms/worker', 'mark ms/iter (worker sum)', - 'sweep ms/iter (worker sum)', 'GCs/iter (worker sum)', 'major/iter (worker sum)', - 'minor/iter (worker sum)', 'global/iter (worker sum)', 'controller compacts/iter*' - ], gc_table[0] - assert_equal ['object-new', '0', '4.000', '4.000', 'N/A', 'N/A', '7.0', '1.0', '3.0', '3.0', '1.0'], gc_table[1] + 'bench', 'ractors', 'GC ms/iter', 'GC ms/worker', 'GCs/iter', 'major/iter', 'minor/iter', 'global/iter', 'compacts*' + ], gc_table[:rows][0] + assert_equal ['object-new', '0', '4.000', '4.000', '7.0', '1.0', '3.0', '3.0', '1.0'], gc_table[:rows][1] + assert_equal ['mark ms/iter', 'sweep ms/iter'], gc_table[:hidden] end end end