From 9db4f307897d1897ada7ae99da655f52751c9080 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 23 Apr 2026 14:18:21 +0800 Subject: [PATCH 01/54] add iterations section for end-2-end config and set for microbenchmarks when creating suites --- src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml | 4 ++++ .../Configurations/InputConfiguration.cs | 2 ++ .../Configurations/Microbenchmarks.Configuration.cs | 2 +- .../Commands/Microbenchmark/MicrobenchmarkCommand.cs | 2 +- .../Commands/RunCommand/BaseSuite/Microbenchmarks.yaml | 2 +- .../Commands/RunCommand/CreateSuiteCommand.cs | 6 ++++++ 6 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml b/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml index 8dac83d30a2..33512caec9f 100644 --- a/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml @@ -12,6 +12,10 @@ coreruns: environment_variables: DOTNET_GCName: clrgc.dll +iteration: + gcperfsim: 1 + microbenchmarks: 1 + trace_configuration_type: gc # Choose between: none, gc, verbose, cpu, cpu_managed, threadtime, join. # Optional fields: the contents of both the symbol_path and the source_path will be copied over to the output path. diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs index fb2546f9bb7..b0ca7613bc6 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs @@ -13,6 +13,8 @@ public sealed class InputConfiguration // public Dictionary? clrgcs { get; set; } // public string? debug_parameters { get; set; } + public Dictionary? iterations { get; set; } + public Dictionary? symbol_path { get; set; } public Dictionary? source_path { get; set; } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs index c44e02cd947..7ea7f6ac4bd 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs @@ -24,7 +24,7 @@ public sealed class Run : RunBase public class Environment { public uint default_max_seconds { get; set; } = 300; - public uint iteration { get; set; } = 1; + public uint iterations { get; set; } = 1; } public sealed class MicrobenchmarkConfigurations diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index ca063389ef5..22ddf41e6c5 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -149,7 +149,7 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi (string, string) fileNameAndCommand = MicrobenchmarkCommandBuilder.Build(configuration, run, benchmark, invocationCountFromBaseline); run.Value.Name = run.Key; - for (int index = 0; index < configuration.Environment.iteration; index++) + for (int index = 0; index < configuration.Environment.iterations; index++) { AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbechmarks: {configuration.Name} - {run.Key} {benchmark} - iteration: {index} [/]\n"); // Run The BDN process with the trace collector. diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml index ea719bad713..257f021c024 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml @@ -6,7 +6,7 @@ microbenchmark_configurations: environment: default_max_seconds: 3000 - iteration: 1 + iterations: 1 # Configurations that involve capturing a trace. trace_configurations: diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs index 0cec453d0c3..96a6943019b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs @@ -225,6 +225,12 @@ internal static MicrobenchmarkConfiguration CreateBaseMicrobenchmarkSuite(InputC }); } + // Set iterations if they exist. + if (inputConfiguration.iterations != null) + { + configuration.Environment.iterations = inputConfiguration.iterations.GetValueOrDefault("microbenchmarks", 1); + } + // The first run is always the baseline. configuration.Runs.First().Value.is_baseline = true; From 980248464a6b04846b64597d1ad3e2c88182b1df Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 24 Apr 2026 17:41:41 +0800 Subject: [PATCH 02/54] add json-trace map and implement AnalyzeForBenchmark --- .../MicrobenchmarkResultsAnalyzer.cs | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs index 4ad9158e74c..c7d5cd72c4c 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs @@ -7,6 +7,129 @@ namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { public static class MicrobenchmarkResultsAnalyzer { + public static Dictionary MapBenchmarkResultAndTraceForBenchmark(string outputPathForRun, string benchmarkTitle) + { + Dictionary benchmarkResultToTraceMap = new(); + + // Extract benchmark title without date + string[] benchmarkTitleParts = benchmarkTitle.Split('-'); + string benchmarkTitleWithoutDate = benchmarkTitleParts[0]; + + string[] jsonFiles = Directory.GetFiles(outputPathForRun, $"{benchmarkTitleWithoutDate}-report-full.json", SearchOption.AllDirectories); + + // Sort JSON files by their parent directory name (timestamp) to ensure consistent ordering + var sortedJsonFiles = jsonFiles + .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) + .ToArray(); + + // TODO: benchmarkTitleWithoutDate doesn't include method name and parameter name + + // Find all trace files for this benchmark + string[] traceFiles = Directory.GetFiles(outputPathForRun, $"{benchmarkTitleWithoutDate}*.etlx", SearchOption.TopDirectoryOnly) + .OrderBy(traceFile => Path.GetFileName(traceFile)) + .ToArray(); + + if (sortedJsonFiles.Length != traceFiles.Length) + { + throw new InvalidOperationException($"The number of JSON files ({sortedJsonFiles.Length}) does not match the number of trace files ({traceFiles.Length}) for benchmark: {benchmarkTitleWithoutDate}. Mapping will be done based on index and may be inaccurate."); + } + + // Map each JSON file to its corresponding trace file based on index + for (int i = 0; i < sortedJsonFiles.Length && i < traceFiles.Length; i++) + { + benchmarkResultToTraceMap[sortedJsonFiles[i]] = traceFiles[i]; + } + + return benchmarkResultToTraceMap; + } + + public static IReadOnlyDictionary>> + AnalyzeForBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkTitle, bool excludeTraces = false) + { + ConcurrentDictionary>> runsToResults = new(); + + Parallel.ForEach(configuration.Runs, (run) => + { + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); + run.Value.Name ??= run.Key; + + // Find the json path for benchmark. + Dictionary benchmarkResultAndTrace = MapBenchmarkResultAndTraceForBenchmark(outputPathForRun, benchmarkTitle); + string[] jsonFiles = benchmarkResultAndTrace.Keys.ToArray(); + + // Retrieve benchmarks from all the JSON files. + Parallel.ForEach(jsonFiles, (jsonFile) => + { + MicrobenchmarkResults results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); + + foreach (var benchmark in results?.Benchmarks) + { + string title = benchmark.FullName; + Statistics statistics = benchmark.Statistics; + + if (!runsToResults.TryGetValue(run.Value, out var perBenchmarkData)) + { + runsToResults[run.Value] = perBenchmarkData = new(); + } + + runsToResults[run.Value].GetValueOrDefault(title, new()); + + MicrobenchmarkResult microbenchmarkResult = new() + { + Statistics = statistics, + Parent = run.Value, + MicrobenchmarkName = title, + }; + + if (!excludeTraces) + { + string tracePath = benchmarkResultAndTrace[jsonFile]; + Analyzer analyzer = AnalyzerManager.GetAnalyzer(tracePath); + + List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); + List corerunProcesses = analyzer.GetProcessGCData("corerun"); + allPertinentProcesses.AddRange(corerunProcesses); + + GCProcessData? benchmarkGCData = null; + foreach (var process in allPertinentProcesses) + { + string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); + string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); + if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) + { + benchmarkGCData = process; + break; + } + } + + if (benchmarkGCData != null) + { + int processID = benchmarkGCData.ProcessID; + microbenchmarkResult.GCData = benchmarkGCData; + microbenchmarkResult.ResultItem = new Presentation.GCPerfSim.ResultItem(benchmarkGCData, tracePath, benchmark.FullName); + /* + TODO: THIS NEEDS TO BE ADDED BACK. + if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) + { + // TODO: Add parameterize. + benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", + symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), + symbolPath: Path.Combine(configuration.Output.Path, run.Key)); + var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); + d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); + benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); + } + */ + } + } + runsToResults[run.Value][title].Add(microbenchmarkResult); + } + }); + }); + + return runsToResults; + } + public static IReadOnlyDictionary> Analyze(MicrobenchmarkConfiguration configuration, bool excludeTraces = false) { ConcurrentDictionary> runsToResults = new(); @@ -174,5 +297,15 @@ public static IReadOnlyList GetComparisons(Micr return comparisonResults; } + + //public static MicrobenchmarkComparisonResults GetComparisonsForBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkTitle, bool excludeTraces = false) + //{ + // IReadOnlyDictionary>> runResultsForBenchmark = AnalyzeForBenchmark(configuration, benchmarkTitle, excludeTraces); + + // MicrobenchmarkComparisonResults comparisonResults = new(); + + + // return comparisonResults; + //} } } From f2318ac0ee87dcf882cd014e5a5603013e5f82f9 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Tue, 28 Apr 2026 15:56:32 +0800 Subject: [PATCH 03/54] Calculate comparison result by benchmark name, rename classes and adjust namespaces --- .../GC.Analysis.API/Statistics.cs | 21 ++ ...robenchmarkResults.cs => BdnJsonResult.cs} | 168 ++++------ .../Analysis/GCTraceMetricComparison.cs | 8 + .../Analysis/GCTraceMetricComparisonResult.cs | 106 ++++++ .../Analysis/GCTraceMetrics.cs | 145 ++++++++ .../MicrobenchmarkComparisonResult.cs | 100 ++++-- .../Microbenchmarks/MicrobenchmarkResult.cs | 57 ++++ .../MicrobenchmarkResultComparison.cs | 225 +++++++++++++ .../MicrobenchmarkResultsAnalyzer.cs | 311 ------------------ .../Microbenchmarks/{Json => }/Json.cs | 2 +- .../Microbenchmarks/Json/JsonOutput.cs | 12 - .../Presentation/Microbenchmarks/Markdown.cs | 2 + .../Microbenchmarks/Presentation.cs | 9 +- .../MicrobenchmarkAnalyzeCommand.cs | 1 - .../Microbenchmark/MicrobenchmarkCommand.cs | 2 +- .../BaseSuite/MicrobenchmarksToRun.txt | 1 - 16 files changed, 698 insertions(+), 472 deletions(-) rename src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/{Microbenchmarks/MicrobenchmarkResults.cs => BdnJsonResult.cs} (67%) create mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs create mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs create mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs create mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs create mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs delete mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs rename src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/{Json => }/Json.cs (87%) delete mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs index 85c8821a8ec..a03cf8fb15b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs @@ -31,5 +31,26 @@ public static double StandardDeviation(this IEnumerable doubleList) double sumOfDerivationAverage = sumOfDerivation / (doubleList.Count() - 1); return Math.Sqrt(sumOfDerivationAverage - (average * average)); } + + public static IEnumerable RemoveOutliers(IEnumerable collection) + { + if (!collection.Any()) + { + return Array.Empty(); + } + // Calculate Q1 (25th percentile) and Q3 (75th percentile) + double q1 = GC.Analysis.API.Statistics.Percentile(collection, 0.25); + double q3 = GC.Analysis.API.Statistics.Percentile(collection, 0.75); + + // Calculate IQR (Interquartile Range) + double iqr = q3 - q1; + + // Calculate bounds: [Q1 - 1.5*IQR, Q3 + 1.5*IQR] + double lowerBound = q1 - 1.5 * iqr; + double upperBound = q3 + 1.5 * iqr; + + // Filter out outliers + return GoodLinq.Where(collection, x => x >= lowerBound && x <= upperBound); + } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs similarity index 67% rename from src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs rename to src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs index 2af39abd257..e228ca6eff4 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs @@ -1,103 +1,10 @@ -using GC.Analysis.API; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using GC.Infrastructure.Core.Presentation.GCPerfSim; -using Newtonsoft.Json; - -namespace GC.Infrastructure.Core.Analysis +namespace GC.Infrastructure.Core.Analysis { - public sealed class MicrobenchmarkResult - { - public Statistics Statistics { get; set; } - - [JsonIgnore] - public GCProcessData? GCData { get; set; } - - public ResultItem ResultItem { get; set; } - - [JsonIgnore] - public CPUProcessData? CPUData { get; set; } - public Run Parent { get; set; } - public string MicrobenchmarkName { get; set; } - public Dictionary OtherMetrics { get; set; } = new(); - - private static readonly IReadOnlyDictionary> _customStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - { "number of iterations", (Statistics stats) => stats.N }, - { "min", (Statistics stats) => stats.Min }, - { "max", (Statistics stats) => stats.Max }, - { "median", (Statistics stats) => stats.Median }, - { "q1", (Statistics stats) => stats.Q1 }, - { "q3", (Statistics stats) => stats.Q3 }, - { "variance", (Statistics stats) => stats.Variance }, - { "standard deviation", (Statistics stats) => stats.StandardDeviation }, - { "skewness", (Statistics stats) => stats.Skewness }, - { "kurtosis", (Statistics stats) => stats.Kurtosis }, - { "standard error", (Statistics stats) => stats.StandardError }, - { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, - }; - - public static double? LookupStatisticsCalculation(string columnName, MicrobenchmarkResult result) - { - if (string.IsNullOrEmpty(columnName)) - { - return null; - } - - if (!_customStatisticsCalculationMap.TryGetValue(columnName, out var val)) - { - return null; - } - - else - { - return val.Invoke(result.Statistics); - } - } - } - - public sealed class Benchmark - { - public string DisplayInfo { get; set; } - public string Namespace { get; set; } - public string Type { get; set; } - public string Method { get; set; } - public string MethodTitle { get; set; } - public string Parameters { get; set; } - public string FullName { get; set; } - public Statistics Statistics { get; set; } - public Memory Memory { get; set; } - public List Measurements { get; set; } - public List Metrics { get; set; } - } - public sealed class ChronometerFrequency { public int Hertz { get; set; } } - public sealed class ConfidenceInterval - { - public int N { get; set; } - public double? Mean { get; set; } - public double? StandardError { get; set; } - public int? Level { get; set; } - public double? Margin { get; set; } - public double? Lower { get; set; } - public double? Upper { get; set; } - } - - public sealed class Descriptor - { - public string Id { get; set; } - public string DisplayName { get; set; } - public string Legend { get; set; } - public string NumberFormat { get; set; } - public int UnitType { get; set; } - public string Unit { get; set; } - public bool TheGreaterTheBetter { get; set; } - public int PriorityInCategory { get; set; } - } - public sealed class HostEnvironmentInfo { public string BenchmarkDotNetCaption { get; set; } @@ -117,6 +24,16 @@ public sealed class HostEnvironmentInfo public string HardwareTimerKind { get; set; } } + + public sealed class Memory + { + public int Gen0Collections { get; set; } + public int Gen1Collections { get; set; } + public int Gen2Collections { get; set; } + public int TotalOperations { get; set; } + public long BytesAllocatedPerOperation { get; set; } + } + public sealed class Measurement { public string IterationMode { get; set; } @@ -127,19 +44,27 @@ public sealed class Measurement public long Nanoseconds { get; set; } } - public sealed class Memory + public sealed class Descriptor { - public int Gen0Collections { get; set; } - public int Gen1Collections { get; set; } - public int Gen2Collections { get; set; } - public int TotalOperations { get; set; } - public long BytesAllocatedPerOperation { get; set; } + public string Id { get; set; } + public string DisplayName { get; set; } + public string Legend { get; set; } + public string NumberFormat { get; set; } + public int UnitType { get; set; } + public string Unit { get; set; } + public bool TheGreaterTheBetter { get; set; } + public int PriorityInCategory { get; set; } } - public sealed class Metric + public sealed class ConfidenceInterval { - public double Value { get; set; } - public Descriptor Descriptor { get; set; } + public int N { get; set; } + public double? Mean { get; set; } + public double? StandardError { get; set; } + public int? Level { get; set; } + public double? Margin { get; set; } + public double? Lower { get; set; } + public double? Upper { get; set; } } public sealed class Percentiles @@ -155,13 +80,6 @@ public sealed class Percentiles public double P100 { get; set; } } - public sealed class MicrobenchmarkResults - { - public string Title { get; set; } - public HostEnvironmentInfo HostEnvironmentInfo { get; set; } - public List Benchmarks { get; set; } - } - public sealed class Statistics { public List OriginalValues { get; set; } @@ -186,4 +104,32 @@ public sealed class Statistics public ConfidenceInterval? ConfidenceInterval { get; set; } public Percentiles Percentiles { get; set; } } -} \ No newline at end of file + + public sealed class Metric + { + public double Value { get; set; } + public Descriptor Descriptor { get; set; } + } + + public sealed class Benchmark + { + public string DisplayInfo { get; set; } + public string Namespace { get; set; } + public string Type { get; set; } + public string Method { get; set; } + public string MethodTitle { get; set; } + public string Parameters { get; set; } + public string FullName { get; set; } + public Statistics Statistics { get; set; } + public Memory Memory { get; set; } + public List Measurements { get; set; } + public List Metrics { get; set; } + } + + public sealed class BdnJsonResult + { + public string Title { get; set; } + public HostEnvironmentInfo HostEnvironmentInfo { get; set; } + public List Benchmarks { get; set; } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs new file mode 100644 index 00000000000..3715fb5657a --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs @@ -0,0 +1,8 @@ +namespace GC.Infrastructure.Core.Analysis +{ + public static class GCTraceMetricComparison + { + public static GCTraceMetricComparisonResult CompareGCTraceMetric(IEnumerable baselines, IEnumerable comparands,string nameOfMetric) + => new GCTraceMetricComparisonResult(baselines, comparands, nameOfMetric); + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs new file mode 100644 index 00000000000..2ca6aadf149 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs @@ -0,0 +1,106 @@ +using GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Analysis.GC; +using System.Reflection; + +namespace GC.Infrastructure.Core.Analysis +{ + public sealed class GCTraceMetricComparisonResult + { + public GCTraceMetricComparisonResult(IEnumerable baselines, IEnumerable comparands, string metricName) + { + Baselines = baselines; + Comparands = comparands; + + MetricName = metricName; + PropertyInfo pInfo = typeof(GCTraceMetrics).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public); + + // Property found on the GCTraceMetrics. + if (pInfo != null) + { + OriginalBaselineMetricCollection = GoodLinq.Select(Baselines, baseline => (double)pInfo.GetValue(baseline)); + OriginalComparandMetricCollection = GoodLinq.Select(Comparands, comparand => (double)pInfo.GetValue(comparand)); + } + + // If property isn't found on the GCTraceMetrics, look in GCStats. + // TODO: Add the case where we look into the map. + else + { + pInfo = typeof(GCStats).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public); + if (pInfo == null) + { + FieldInfo fieldInfo = typeof(GCStats).GetField(metricName, BindingFlags.Instance | BindingFlags.Public); + if (fieldInfo == null) + { + // Out of luck! + OriginalBaselineMetricCollection = Array.Empty(); + OriginalComparandMetricCollection = Array.Empty(); + OutliersFreeBaselineMetricCollection = Array.Empty(); + OutliersFreeComparandMetricCollection = Array.Empty(); + AveragedBaselineMetric = double.NaN; + AveragedComparandMetric = double.NaN; + return; + } + + else + { + OriginalBaselineMetricCollection = GoodLinq.Select(Baselines, baseline => (double)fieldInfo.GetValue(baseline)); + OriginalComparandMetricCollection = GoodLinq.Select(Comparands, comparand => (double)fieldInfo.GetValue(comparand)); + } + } + + else + { + OriginalBaselineMetricCollection = GoodLinq.Select(Baselines, baseline => (double)pInfo.GetValue(baseline)); + OriginalComparandMetricCollection = GoodLinq.Select(Comparands, comparand => (double)pInfo.GetValue(comparand)); + } + } + + // Filter out outliers using IQR method + OutliersFreeBaselineMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(OriginalBaselineMetricCollection); + OutliersFreeComparandMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(OriginalComparandMetricCollection); + + // Calculate averaged metrics + AveragedBaselineMetric = OutliersFreeBaselineMetricCollection.Any() + ? OutliersFreeBaselineMetricCollection.Average() + : double.NaN; + AveragedComparandMetric = OutliersFreeComparandMetricCollection.Any() + ? OutliersFreeComparandMetricCollection.Average() + : double.NaN; + } + + public string RunName => Baselines.First().RunName; + public string MetricName { get; } + public IEnumerable OriginalBaselineMetricCollection { get; } + public IEnumerable OriginalComparandMetricCollection { get; } + public IEnumerable OutliersFreeBaselineMetricCollection { get; } + public IEnumerable OutliersFreeComparandMetricCollection { get; } + + public double AveragedBaselineMetric { get; } + public double AveragedComparandMetric { get; } + public double Delta => AveragedComparandMetric - AveragedBaselineMetric; + public double PercentageDelta + { + get + { + if (AveragedBaselineMetric == 0) + { + if (AveragedComparandMetric == 0) + { + return 0; + } + else + { + return double.NaN; + } + } + else + { + return Delta / AveragedBaselineMetric * 100.0; + } + } + } + public string Key => $"{Baselines.FirstOrDefault()?.ConfigurationName}_{RunName}"; + public IEnumerable Baselines { get; } + public IEnumerable Comparands { get; } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs new file mode 100644 index 00000000000..fb0deabc532 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs @@ -0,0 +1,145 @@ +using GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; + +namespace GC.Infrastructure.Core.Analysis +{ + public sealed class GCTraceMetrics + { + public static GCTraceMetrics GetNullItem(string runName, string corerun) => + new GCTraceMetrics(runName, corerun); + + private GCTraceMetrics(string runName, string corerun) + { + ConfigurationName = corerun; + RunName = runName; + PctTimePausedInGC = double.NaN; + FirstToLastGCSeconds = double.NaN; + HeapSizeBeforeMB_Mean = double.NaN; + HeapSizeAfter_Mean = double.NaN; + TotalCommittedInUse = double.NaN; + TotalBookkeepingCommitted = double.NaN; + TotalCommittedInGlobalDecommit = double.NaN; + TotalCommittedInFree = double.NaN; + TotalCommittedInGlobalFree = double.NaN; + PauseDurationMSec_95PWhereIsGen0 = double.NaN; + PauseDurationMSec_95PWhereIsGen1 = double.NaN; + PauseDurationMSec_95PWhereIsBackground = double.NaN; + PauseDurationMSec_MeanWhereIsBackground = double.NaN; + PauseDurationMSec_95PWhereIsBlockingGen2 = double.NaN; + PauseDurationMSec_MeanWhereIsBlockingGen2 = double.NaN; + CountIsBlockingGen2 = double.NaN; + PauseDurationSeconds_SumWhereIsGen1 = double.NaN; + PauseDurationMSec_MeanWhereIsEphemeral = double.NaN; + PromotedMB_MeanWhereIsGen1 = double.NaN; + CountIsGen1 = double.NaN; + CountIsGen0 = double.NaN; + HeapCount = double.NaN; + PauseDurationMSec_Sum = double.NaN; + TotalAllocatedMB = double.NaN; + TotalNumberGCs = double.NaN; + Speed_MBPerMSec = double.NaN; + ExecutionTimeMSec = double.NaN; + } + + public GCTraceMetrics(GCProcessData processData, string runName, string configurationName) + { + RunName = runName; + ConfigurationName = configurationName; + ExecutionTimeMSec = processData.DurationMSec; + + PctTimePausedInGC = processData.Stats.GetGCPauseTimePercentage(); + FirstToLastGCSeconds = (processData.GCs.Last().StartRelativeMSec - processData.GCs.First().StartRelativeMSec) / 1000; + HeapSizeAfter_Mean = GoodLinq.Average(processData.GCs, (gc => gc.HeapSizeAfterMB)); + HeapSizeBeforeMB_Mean = GoodLinq.Average(processData.GCs, (gc => gc.HeapSizeBeforeMB)); + + TotalCommittedInUse = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInUse)); + TotalBookkeepingCommitted = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalBookkeepingCommitted)); + TotalCommittedInGlobalDecommit = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInGlobalDecommit)); + TotalCommittedInFree = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInFree)); + TotalCommittedInGlobalFree = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInGlobalFree)); + + var properties = processData.Stats.GetType().GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); + foreach (var property in properties) + { + if (property.PropertyType != typeof(double) || property.PropertyType != typeof(int)) + { + continue; + } + + string propertyName = property.Name; + double propertyValue = (double)(property.GetValue(processData.Stats) ?? double.NaN); + StatsData[propertyName] = propertyValue; + } + + var fields = processData.Stats.GetType().GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); + foreach (var field in fields) + { + if (field.FieldType != typeof(double) || field.FieldType != typeof(int)) + { + continue; + } + + string name = field.Name; + double value = (double)(field.GetValue(processData.Stats) ?? double.NaN); + StatsData[name] = value; + } + + // 95P + PauseDurationMSec_95PWhereIsGen0 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 0)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_95PWhereIsGen1 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PauseDurationMSec)), 0.95); + + PauseDurationMSec_95PWhereIsBackground = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type == GCType.BackgroundGC)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_MeanWhereIsBackground = GoodLinq.Average(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type == GCType.BackgroundGC)), (gc => gc.PauseDurationMSec)), (p => p)); + + PauseDurationMSec_95PWhereIsBlockingGen2 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type != GCType.BackgroundGC && gc.Generation == 2)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_MeanWhereIsBlockingGen2 = GoodLinq.Average(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type != GCType.BackgroundGC && gc.Generation == 2)), (gc => gc.PauseDurationMSec)), (p => p)); + + CountIsBlockingGen2 = processData.GCs.Count(gc => gc.Generation == 2 && gc.Type != GCType.BackgroundGC); + + HeapCount = processData.Stats.HeapCount; + TotalNumberGCs = processData.Stats.Count; + TotalAllocatedMB = processData.Stats.TotalAllocatedMB; + + Speed_MBPerMSec = processData.Stats.TotalPromotedMB / processData.Stats.TotalPauseTimeMSec; + + PauseDurationMSec_MeanWhereIsEphemeral = + GoodLinq.Average(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1 || gc.Generation == 0)), (gc => gc.PauseDurationMSec)); + PauseDurationSeconds_SumWhereIsGen1 = + GoodLinq.Sum(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PauseDurationMSec)); + PauseDurationMSec_Sum = GoodLinq.Sum(processData.GCs, (gc => gc.PauseDurationMSec)); + CountIsGen1 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 1).Count; + CountIsGen0 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 0).Count; + } + + public double PctTimePausedInGC { get; } + public double FirstToLastGCSeconds { get; } + public double HeapSizeBeforeMB_Mean { get; } + public double HeapSizeAfter_Mean { get; } + public double TotalCommittedInUse { get; set; } + public double TotalCommittedInGlobalDecommit { get; set; } + public double TotalCommittedInFree { get; set; } + public double TotalCommittedInGlobalFree { get; set; } + public double TotalBookkeepingCommitted { get; set; } + public double PauseDurationMSec_95PWhereIsGen0 { get; } + public double PauseDurationMSec_95PWhereIsGen1 { get; } + public double PauseDurationMSec_95PWhereIsBackground { get; } + public double PauseDurationMSec_MeanWhereIsBackground { get; } + public double PauseDurationMSec_95PWhereIsBlockingGen2 { get; } + public double PauseDurationMSec_MeanWhereIsBlockingGen2 { get; } + public double CountIsBlockingGen2 { get; } + public double PauseDurationSeconds_SumWhereIsGen1 { get; } + public double PauseDurationMSec_MeanWhereIsEphemeral { get; } + public double PromotedMB_MeanWhereIsGen1 { get; } + public double CountIsGen1 { get; } + public double CountIsGen0 { get; } + public double HeapCount { get; } + public double PauseDurationMSec_Sum { get; } + public double TotalAllocatedMB { get; set; } + public double TotalNumberGCs { get; } + public double Speed_MBPerMSec { get; } + public string RunName { get; } + public string ConfigurationName { get; } + public double ExecutionTimeMSec { get; } + public Dictionary StatsData { get; } = new(); + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 60150f4d4c9..eda6072ad80 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -1,4 +1,4 @@ -using GC.Infrastructure.Core.Presentation.GCPerfSim; +using GC.Analysis.API; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { @@ -7,54 +7,94 @@ public sealed class MicrobenchmarkComparisonResult { public MicrobenchmarkComparisonResult() { } - public MicrobenchmarkComparisonResult(MicrobenchmarkResult baseline, MicrobenchmarkResult comparand) + public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands) { - Baseline = baseline; - Comparand = comparand; - var result = new ResultItemComparison(baseline.ResultItem, comparand.ResultItem); + Baselines = baselines; + Comparands = comparands; + var baselineGCTraceMetricsCollection = GoodLinq.Select(Baselines, baseline => baseline.GCTraceMetrics); + var comparandGCTraceMetricsCollection = GoodLinq.Select(Comparands, comparand => comparand.GCTraceMetrics); + + string[] metricNames = new string[] + { + "PctTimePausedInGC", + "ExecutionTimeMSec", + "PauseDurationMSec_MeanWhereIsEphemeral", + "PauseDurationMSec_MeanWhereIsBackground", + "PauseDurationMSec_MeanWhereIsBlockingGen2" + }; + ComparisonResults = new(); - ComparisonResults.Add(result.GetComparison("PctTimePausedInGC")); - ComparisonResults.Add(result.GetComparison("ExecutionTimeMSec")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsEphemeral")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsBackground")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsBlockingGen2")); + foreach (var metricName in metricNames) + { + ComparisonResults.Add( + GCTraceMetricComparison.CompareGCTraceMetric(baselineGCTraceMetricsCollection, comparandGCTraceMetricsCollection, metricName)); + } + + BaselineRunName = Baselines?.FirstOrDefault()?.Parent?.Name; + ComparandRunName = Comparands?.FirstOrDefault()?.Parent?.Name; + MicrobenchmarkName = Baselines?.FirstOrDefault()?.MicrobenchmarkName; + + OriginalBaselineMeanValueCollection = + GoodLinq.Select(Baselines, baseline => baseline.Statistics?.Mean ?? double.NaN).ToArray(); + OriginalComparandMeanValueCollection = + GoodLinq.Select(Comparands, comparand => comparand.Statistics?.Mean ?? double.NaN).ToArray(); } - public MicrobenchmarkResult Baseline { get; set; } - public MicrobenchmarkResult Comparand { get; set; } - public List ComparisonResults { get; set; } + public IEnumerable Baselines { get; set; } + public IEnumerable Comparands { get; set; } + public List ComparisonResults { get; set; } + public string BaselineRunName { get; } + public string ComparandRunName { get; } + public string MicrobenchmarkName { get; } + public double[] OriginalBaselineMeanValueCollection { get; } + public double[] OriginalComparandMeanValueCollection { get; } - // TODO: Nullable double check. - public string BaselineRunName => Baseline?.Parent?.Name; - public string ComparandRunName => Comparand?.Parent?.Name; - public string MicrobenchmarkName => Baseline.MicrobenchmarkName; + public double[] OutliersFreeBaselineMeanValueCollection => + GC.Analysis.API.Statistics.RemoveOutliers(OriginalBaselineMeanValueCollection).ToArray(); + public double[] OutliersFreeComparandMeanValueCollection => + GC.Analysis.API.Statistics.RemoveOutliers(OriginalComparandMeanValueCollection).ToArray(); - public double MeanDiff => (Comparand.Statistics?.Mean.Value - Baseline.Statistics?.Mean.Value) ?? double.NaN; - public double MeanDiffPerc => (MeanDiff / Baseline.Statistics?.Mean.Value) * 100 ?? double.NaN; + public double MeanDiff => OutliersFreeComparandMeanValueCollection.Average() - OutliersFreeBaselineMeanValueCollection.Average(); + public double MeanDiffPerc => (MeanDiff / (Baselines.FirstOrDefault()?.Statistics?.Mean ?? double.NaN)) * 100; - public double? GetDiffPercentFromOtherMetrics(string metric) + public double? GetDiffPercentFromOtherMetrics(string metricName) { - if (!Baseline.OtherMetrics.TryGetValue(metric, out var baselineMetric)) - { - return null; - } + List baselineOtherMetricCollection = new(); + List comparandOtherMetricCollection = new(); - if (!baselineMetric.HasValue) + foreach (var baseline in Baselines) { - return null; + if (baseline.OtherMetrics.TryGetValue(metricName, out var baselineMetric)) + { + if (baselineMetric.HasValue) + { + baselineOtherMetricCollection.Add(baselineMetric.Value); + } + } } - if (!Comparand.OtherMetrics.TryGetValue(metric, out var comparandMetric)) + foreach (var comparand in Comparands) { - return null; + if (comparand.OtherMetrics.TryGetValue(metricName, out var comparandMetric)) + { + if (comparandMetric.HasValue) + { + comparandOtherMetricCollection.Add(comparandMetric.Value); + } + } } - if (!comparandMetric.HasValue) + if (baselineOtherMetricCollection.Count() * comparandOtherMetricCollection.Count() == 0) { return null; } - return (comparandMetric.Value - baselineMetric.Value) / baselineMetric.Value; + var outliersFreeBaselineOtherMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(baselineOtherMetricCollection); + var outliersFreeComparandOtherMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(comparandOtherMetricCollection); + + var averagedOutliersFreeBaselineOtherMetric = outliersFreeBaselineOtherMetricCollection.Average(); + var averagedOutliersFreeComparandOtherMetric = outliersFreeComparandOtherMetricCollection.Average(); + return (averagedOutliersFreeBaselineOtherMetric - averagedOutliersFreeComparandOtherMetric) / averagedOutliersFreeBaselineOtherMetric; } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs new file mode 100644 index 00000000000..9c24753d7c9 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -0,0 +1,57 @@ +using GC.Analysis.API; +using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using Newtonsoft.Json; + +namespace GC.Infrastructure.Core.Analysis +{ + public sealed class MicrobenchmarkResult + { + public Statistics Statistics { get; set; } + + [JsonIgnore] + public GCProcessData? GCData { get; set; } + + public GCTraceMetrics GCTraceMetrics { get; set; } + + [JsonIgnore] + public CPUProcessData? CPUData { get; set; } + public Run Parent { get; set; } + public string MicrobenchmarkName { get; set; } + public Dictionary OtherMetrics { get; set; } = new(); + + private static readonly IReadOnlyDictionary> _customStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "number of iterations", (Statistics stats) => stats.N }, + { "min", (Statistics stats) => stats.Min }, + { "max", (Statistics stats) => stats.Max }, + { "median", (Statistics stats) => stats.Median }, + { "q1", (Statistics stats) => stats.Q1 }, + { "q3", (Statistics stats) => stats.Q3 }, + { "variance", (Statistics stats) => stats.Variance }, + { "standard deviation", (Statistics stats) => stats.StandardDeviation }, + { "skewness", (Statistics stats) => stats.Skewness }, + { "kurtosis", (Statistics stats) => stats.Kurtosis }, + { "standard error", (Statistics stats) => stats.StandardError }, + { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, + }; + + public static double? LookupStatisticsCalculation(string columnName, MicrobenchmarkResult result) + { + if (string.IsNullOrEmpty(columnName)) + { + return null; + } + + if (!_customStatisticsCalculationMap.TryGetValue(columnName, out var val)) + { + return null; + } + + else + { + return val.Invoke(result.Statistics); + } + } + } + +} \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs new file mode 100644 index 00000000000..0dfe503b26a --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -0,0 +1,225 @@ +using GC.Analysis.API; +using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using Newtonsoft.Json; +using System.Collections.Concurrent; + +namespace GC.Infrastructure.Core.Analysis.Microbenchmarks +{ + public static class MicrobenchmarkResultComparison + { + private static readonly Dictionary _benchmarkNameToTraceFilePatternMap = new() + { + { "ByteMark.BenchBitOps", "ByteMark.BenchBitOps"}, + { "System.Collections.CtorGivenSize.Array(Size: 512)", "System.Collections.CtorGivenSize_String_.Array_size_512_"}, + { "System.Collections.Tests.Perf_BitArray.BitArrayByteArrayCtor(Size: 512)", "System.Collections.Tests.Perf_BitArray.BitArrayByteArrayCtor_size_512_"}, + { "System.IO.Tests.Perf_File.ReadAllBytes(size: 104857600)", "System.IO.Tests.Perf_File.ReadAllBytes_size_104857600_"}, + { "System.IO.Tests.Perf_File.ReadAllBytesAsync(size: 104857600)", "System.IO.Tests.Perf_File.ReadAllBytesAsync_size_104857600_"}, + { "System.Linq.Tests.Perf_Enumerable.ToArray(input: ICollection)", "System.Linq.Tests.Perf_Enumerable.ToArray_"}, + { "System.Linq.Tests.Perf_Enumerable.ToArray(input: IEnumerable)", "System.Linq.Tests.Perf_Enumerable.ToArray_"}, + { "System.Numerics.Tests.Perf_BigInteger.Add(arguments: 65536,65536 bits)", "System.Numerics.Tests.Perf_BigInteger.Add_arguments_65536_"}, + { "System.Numerics.Tests.Perf_BigInteger.Subtract(arguments: 65536,65536 bits)", "System.Numerics.Tests.Perf_BigInteger.Subtract_arguments_65536_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 1000)", "System.Tests.Perf_GC_Byte_.NewOperator_Array_length_1000_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Byte_.NewOperator_Array_length_10000_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 1000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_1000_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_10000_"}, + }; + + private static readonly ConcurrentDictionary>> _benchmarkFullNameToJsonForRun = new(); + + public static ConcurrentDictionary> MapBenchmarkFullNameToJsonForRun(string outputPathForRun) + { + return _benchmarkFullNameToJsonForRun.GetOrAdd(outputPathForRun, path => + { + ConcurrentDictionary> benchmarkFullNameJsonMap = new(); + + string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); + + Parallel.ForEach(jsonFiles, (jsonFile) => { + BdnJsonResult results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); + string fullName = results.Benchmarks.FirstOrDefault()?.FullName; + benchmarkFullNameJsonMap[fullName] = benchmarkFullNameJsonMap.GetValueOrDefault(fullName, new()); + benchmarkFullNameJsonMap[fullName].Add(jsonFile); + }); + + return benchmarkFullNameJsonMap; + }); + } + + public static ConcurrentDictionary MapJsonToTraceForSingleBenchmarkRun(string outputPathForRun, string benchmarkFullName) + { + ConcurrentDictionary jsonTraceMap = new(); + + var benchmarkFullNameJsonMap = MapBenchmarkFullNameToJsonForRun(outputPathForRun); + + string[] jsonFiles = benchmarkFullNameJsonMap.GetValueOrDefault(benchmarkFullName, new()).ToArray(); + + Parallel.ForEach(jsonFiles, (jsonFile) => { + BdnJsonResult results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); + string fullName = results.Benchmarks.FirstOrDefault()?.FullName; + if (fullName != benchmarkFullName) + { + return; + } + // placeholder + jsonTraceMap[jsonFile] = ""; + }); + + string[] sortedJsonFiles = jsonTraceMap.Keys + .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) + .ToArray(); + + string traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkFullName]; + + string[] sortedTraceFiles = Enumerable.Where(Directory.GetFiles(outputPathForRun, "*.etlx", SearchOption.TopDirectoryOnly), traceFile => + Path.GetFileName(traceFile).ToLower().Contains(traceFileNameTemplate.ToLower())) + .OrderBy(traceFile => traceFile) + .ToArray(); + + if (sortedJsonFiles.Length != sortedTraceFiles.Length) + { + throw new InvalidOperationException( + $"The number of JSON files ({sortedJsonFiles.Length}) does not match the number of trace files ({sortedTraceFiles.Length}) for benchmark: {benchmarkFullName}"); + } + + for (int idx = 0; idx < sortedJsonFiles.Length; idx++) + { + jsonTraceMap[sortedJsonFiles[idx]] = sortedTraceFiles[idx]; + } + + return jsonTraceMap; + } + + public static IReadOnlyDictionary> AnalyzeMicrobenchmarkResultsForSingleBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) + { + ConcurrentDictionary> runsToResults = new(); + + Parallel.ForEach(configuration.Runs, (run) => + { + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); + run.Value.Name ??= run.Key; + + var jsonTraceMap = MapJsonToTraceForSingleBenchmarkRun(outputPathForRun, benchmarkFullName); + + runsToResults[run.Value] = runsToResults.GetValueOrDefault(run.Value, new()); + + Parallel.ForEach(jsonTraceMap, jsonTracePair => { + string jsonPath = jsonTracePair.Key; + string tracePath = jsonTracePair.Value; + + BdnJsonResult results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); + + foreach (var benchmark in results?.Benchmarks) + { + Statistics statistics = benchmark.Statistics; + + MicrobenchmarkResult microbenchmarkResult = new() + { + Statistics = statistics, + Parent = run.Value, + MicrobenchmarkName = benchmarkFullName, + }; + + if (!excludeTraces) + { + Analyzer analyzer = AnalyzerManager.GetAnalyzer(tracePath); + + List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); + List corerunProcesses = analyzer.GetProcessGCData("corerun"); + allPertinentProcesses.AddRange(corerunProcesses); + + GCProcessData? benchmarkGCData = null; + foreach (var process in allPertinentProcesses) + { + string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); + string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); + if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) + { + benchmarkGCData = process; + break; + } + } + + if (benchmarkGCData != null) + { + int processID = benchmarkGCData.ProcessID; + microbenchmarkResult.GCData = benchmarkGCData; + microbenchmarkResult.GCTraceMetrics = new GCTraceMetrics(benchmarkGCData, tracePath, benchmark.FullName); + /* + TODO: THIS NEEDS TO BE ADDED BACK. + if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) + { + // TODO: Add parameterize. + benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", + symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), + symbolPath: Path.Combine(configuration.Output.Path, run.Key)); + var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); + d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); + benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); + } + */ + } + } + runsToResults[run.Value].Add(microbenchmarkResult); + } + }); + + }); + + return runsToResults; + } + + public static List CompareMicrobenchmarkResultForBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) + { + IReadOnlyDictionary> runResults = AnalyzeMicrobenchmarkResultsForSingleBenchmark(configuration, benchmarkFullName, excludeTraces); + List comparisonResults = new(); + + if (configuration.Output.run_comparisons != null) + { + foreach (var comparison in configuration.Output.run_comparisons) + { + string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); + string baselineName = breakup[0]; + string runName = breakup[1]; + + var baselineRuns = GoodLinq.Where(runResults.Keys, r => r.Name == baselineName); + var comparandRuns = GoodLinq.Where(runResults.Keys, r => r.Name == runName); + + var baselineMicrobenchmarkResults = GoodLinq.Select(baselineRuns, b => runResults[b]).SelectMany(r => r); + var comparandMicrobenchmarkResults = GoodLinq.Select(comparandRuns, c => runResults[c]).SelectMany(r => r); + + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults)); + } + } + + // Default case where the run comparisons aren't specified. + else + { + var baselineRuns = GoodLinq.Where(runResults.Keys, r => r.is_baseline); + var comparandRuns = GoodLinq.Where(runResults.Keys, r => !r.is_baseline); + + var baselineMicrobenchmarkResults = GoodLinq.Select(baselineRuns, b => runResults[b]).SelectMany(r => r); + var comparandMicrobenchmarkResults = GoodLinq.Select(comparandRuns, c => runResults[c]).SelectMany(r => r); + + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults)); + } + + return comparisonResults; + } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs deleted file mode 100644 index c7d5cd72c4c..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs +++ /dev/null @@ -1,311 +0,0 @@ -using GC.Analysis.API; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using Newtonsoft.Json; -using System.Collections.Concurrent; - -namespace GC.Infrastructure.Core.Analysis.Microbenchmarks -{ - public static class MicrobenchmarkResultsAnalyzer - { - public static Dictionary MapBenchmarkResultAndTraceForBenchmark(string outputPathForRun, string benchmarkTitle) - { - Dictionary benchmarkResultToTraceMap = new(); - - // Extract benchmark title without date - string[] benchmarkTitleParts = benchmarkTitle.Split('-'); - string benchmarkTitleWithoutDate = benchmarkTitleParts[0]; - - string[] jsonFiles = Directory.GetFiles(outputPathForRun, $"{benchmarkTitleWithoutDate}-report-full.json", SearchOption.AllDirectories); - - // Sort JSON files by their parent directory name (timestamp) to ensure consistent ordering - var sortedJsonFiles = jsonFiles - .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) - .ToArray(); - - // TODO: benchmarkTitleWithoutDate doesn't include method name and parameter name - - // Find all trace files for this benchmark - string[] traceFiles = Directory.GetFiles(outputPathForRun, $"{benchmarkTitleWithoutDate}*.etlx", SearchOption.TopDirectoryOnly) - .OrderBy(traceFile => Path.GetFileName(traceFile)) - .ToArray(); - - if (sortedJsonFiles.Length != traceFiles.Length) - { - throw new InvalidOperationException($"The number of JSON files ({sortedJsonFiles.Length}) does not match the number of trace files ({traceFiles.Length}) for benchmark: {benchmarkTitleWithoutDate}. Mapping will be done based on index and may be inaccurate."); - } - - // Map each JSON file to its corresponding trace file based on index - for (int i = 0; i < sortedJsonFiles.Length && i < traceFiles.Length; i++) - { - benchmarkResultToTraceMap[sortedJsonFiles[i]] = traceFiles[i]; - } - - return benchmarkResultToTraceMap; - } - - public static IReadOnlyDictionary>> - AnalyzeForBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkTitle, bool excludeTraces = false) - { - ConcurrentDictionary>> runsToResults = new(); - - Parallel.ForEach(configuration.Runs, (run) => - { - string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); - run.Value.Name ??= run.Key; - - // Find the json path for benchmark. - Dictionary benchmarkResultAndTrace = MapBenchmarkResultAndTraceForBenchmark(outputPathForRun, benchmarkTitle); - string[] jsonFiles = benchmarkResultAndTrace.Keys.ToArray(); - - // Retrieve benchmarks from all the JSON files. - Parallel.ForEach(jsonFiles, (jsonFile) => - { - MicrobenchmarkResults results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); - - foreach (var benchmark in results?.Benchmarks) - { - string title = benchmark.FullName; - Statistics statistics = benchmark.Statistics; - - if (!runsToResults.TryGetValue(run.Value, out var perBenchmarkData)) - { - runsToResults[run.Value] = perBenchmarkData = new(); - } - - runsToResults[run.Value].GetValueOrDefault(title, new()); - - MicrobenchmarkResult microbenchmarkResult = new() - { - Statistics = statistics, - Parent = run.Value, - MicrobenchmarkName = title, - }; - - if (!excludeTraces) - { - string tracePath = benchmarkResultAndTrace[jsonFile]; - Analyzer analyzer = AnalyzerManager.GetAnalyzer(tracePath); - - List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); - List corerunProcesses = analyzer.GetProcessGCData("corerun"); - allPertinentProcesses.AddRange(corerunProcesses); - - GCProcessData? benchmarkGCData = null; - foreach (var process in allPertinentProcesses) - { - string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); - string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); - if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) - { - benchmarkGCData = process; - break; - } - } - - if (benchmarkGCData != null) - { - int processID = benchmarkGCData.ProcessID; - microbenchmarkResult.GCData = benchmarkGCData; - microbenchmarkResult.ResultItem = new Presentation.GCPerfSim.ResultItem(benchmarkGCData, tracePath, benchmark.FullName); - /* - TODO: THIS NEEDS TO BE ADDED BACK. - if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) - { - // TODO: Add parameterize. - benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", - symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), - symbolPath: Path.Combine(configuration.Output.Path, run.Key)); - var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); - d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); - benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); - } - */ - } - } - runsToResults[run.Value][title].Add(microbenchmarkResult); - } - }); - }); - - return runsToResults; - } - - public static IReadOnlyDictionary> Analyze(MicrobenchmarkConfiguration configuration, bool excludeTraces = false) - { - ConcurrentDictionary> runsToResults = new(); - - Parallel.ForEach(configuration.Runs, (run) => - { - string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); - run.Value.Name ??= run.Key; - - // Find the json path. - string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); - - // Retrieve benchmarks from all the JSON files. - Parallel.ForEach(jsonFiles, (jsonFile) => - { - MicrobenchmarkResults results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); - foreach (var benchmark in results?.Benchmarks) - { - string title = benchmark.FullName; - Statistics statistics = benchmark.Statistics; - - if (!runsToResults.TryGetValue(run.Value, out var perBenchmarkData)) - { - runsToResults[run.Value] = perBenchmarkData = new ConcurrentDictionary(); - } - - runsToResults[run.Value][title] = new MicrobenchmarkResult - { - Statistics = statistics, - Parent = run.Value, - MicrobenchmarkName = title, - }; - } - }); - - if (!excludeTraces) - { - Dictionary analyzers = AnalyzerManager.GetAllAnalyzers(outputPathForRun); - - foreach (var analyzer in analyzers) - { - List allPertinentProcesses = analyzer.Value.GetProcessGCData("dotnet"); - List corerunProcesses = analyzer.Value.GetProcessGCData("corerun"); - allPertinentProcesses.AddRange(corerunProcesses); - foreach (var benchmark in runsToResults[run.Value]) - { - GCProcessData? benchmarkGCData = null; - foreach (var process in allPertinentProcesses) - { - string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); - string runCleaned = benchmark.Key.Replace("\"", "").Replace("\\", ""); - if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) - { - benchmarkGCData = process; - break; - } - } - - if (benchmarkGCData != null) - { - int processID = benchmarkGCData.ProcessID; - benchmark.Value.GCData = benchmarkGCData; - benchmark.Value.ResultItem = new Presentation.GCPerfSim.ResultItem(benchmarkGCData, analyzer.Key, benchmark.Key); - /* - TODO: THIS NEEDS TO BE ADDED BACK. - if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) - { - // TODO: Add parameterize. - benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", - symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), - symbolPath: Path.Combine(configuration.Output.Path, run.Key)); - var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); - d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); - benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); - } - */ - } - } - }; - } - }); - - return runsToResults; - } - - public static IReadOnlyList GetComparisons(MicrobenchmarkConfiguration configuration, bool excludeTraces = false) - { - IReadOnlyDictionary> runResults = Analyze(configuration, excludeTraces); - List comparisonResults = new(); - - if (configuration.Output.run_comparisons != null) - { - foreach (var comparison in configuration.Output.run_comparisons) - { - string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); - string baselineName = breakup[0]; - string runName = breakup[1]; - - Run run = runResults.Keys.FirstOrDefault(k => string.CompareOrdinal(k.Name, runName) == 0); - Run baselineRun = runResults.Keys.FirstOrDefault(k => string.CompareOrdinal(k.Name, baselineName) == 0); - - List microbenchmarkResults = new(); - - // Go through all the microbenchmarks for the current run and find the corresponding runs in the baseline. - foreach (var r in runResults[run]) - { - string microbenchmarkName = r.Key; - if (runResults[baselineRun].TryGetValue(microbenchmarkName, out var m)) - { - MicrobenchmarkComparisonResult microbenchmarkResult = new(m, r.Value); - microbenchmarkResults.Add(microbenchmarkResult); - } - - else - { - // TODO: Log the fact that we haven't found a corresponding result in the baseline. - Console.WriteLine($"Microbenchmark: {microbenchmarkName} isn't found on the baseline: {baselineName} for run: {runName}"); - } - } - - // At this point of time, the lack thereof of either of the runs should be a non-issue. - comparisonResults.Add(new MicrobenchmarkComparisonResults(baselineName, runName, microbenchmarkResults)); - } - } - - // Default case where the run comparisons aren't specified. - else - { - string baselineName = configuration.Runs.FirstOrDefault(r => r.Value.is_baseline).Key; - KeyValuePair> baselineResult = baselineName != null ? runResults.First(r => r.Key.Name == baselineName) : runResults.First(); - - // For each run, we want to grab it and it's baseline and then do a per microbenchmark association. - foreach (var runResult in runResults) - { - Run run = runResult.Key; - string runName = run.Name; - - if (string.CompareOrdinal(runName, baselineName) == 0) - { - continue; - } - - List microbenchmarkResults = new(); - - // Go through all the microbenchmarks for the current run and find the corresponding runs in the baseline. - foreach (var r in runResult.Value) - { - string microbenchmarkName = r.Key; - if (baselineResult.Value.TryGetValue(microbenchmarkName, out var m)) - { - MicrobenchmarkComparisonResult microbenchmarkResult = new(m, r.Value); - microbenchmarkResults.Add(microbenchmarkResult); - } - - else - { - Console.WriteLine($"Microbenchmark: {microbenchmarkName} isn't found on the baseline: {baselineName} for run: {runName}"); - // TODO: Log the fact that we haven't found a corresponding result in the baseline. - } - } - - comparisonResults.Add(new MicrobenchmarkComparisonResults(baselineName, runName, microbenchmarkResults)); - } - } - - return comparisonResults; - } - - //public static MicrobenchmarkComparisonResults GetComparisonsForBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkTitle, bool excludeTraces = false) - //{ - // IReadOnlyDictionary>> runResultsForBenchmark = AnalyzeForBenchmark(configuration, benchmarkTitle, excludeTraces); - - // MicrobenchmarkComparisonResults comparisonResults = new(); - - - // return comparisonResults; - //} - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs similarity index 87% rename from src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs rename to src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs index 3d547c03a0c..3b2e4eeef4f 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs @@ -2,7 +2,7 @@ using GC.Infrastructure.Core.Configurations.Microbenchmarks; using Newtonsoft.Json; -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks.Json +namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Json { diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs deleted file mode 100644 index 9840432b2ba..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks.Json -{ - public sealed class JsonOutput - { - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index 76dffd7f87d..d1237a99d86 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -8,6 +8,7 @@ namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Markdown { + /* private const string baseTableString = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; private const string baseTableRows = "| --- | --- | -- | --- | --- | --- | --- | "; @@ -234,5 +235,6 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm comparison.Comparand?.CPUData?.Parent?.Analyzer?.Dispose(); } } + */ } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs index 93f4faea7f0..6a3a426c532 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs @@ -1,6 +1,5 @@ using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; namespace GC.Infrastructure.Core.Presentation.Microbenchmarks @@ -9,18 +8,20 @@ public static class Presentation { public static IReadOnlyList Present(MicrobenchmarkConfiguration configuration, Dictionary executionDetails) { - IReadOnlyList comparisonResults = MicrobenchmarkResultsAnalyzer.GetComparisons(configuration); + //IReadOnlyList comparisonResults = MicrobenchmarkResultComparison.CompareMicrobenchmarkResults(configuration); + IReadOnlyList comparisonResults = Array.Empty().ToList(); + foreach (var format in configuration.Output.Formats) { if (format == "markdown") { - Markdown.GenerateTable(configuration, comparisonResults, executionDetails, Path.Combine(configuration.Output.Path, "Results.md")); + //Markdown.GenerateTable(configuration, comparisonResults, executionDetails, Path.Combine(configuration.Output.Path, "Results.md")); continue; } if (format == "json") { - Json.Json.Generate(configuration, comparisonResults, Path.Combine(configuration.Output.Path, "Results.json")); + //Json.Json.Generate(configuration, comparisonResults, Path.Combine(configuration.Output.Path, "Results.json")); continue; } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index 35c4e50a266..976e75c07aa 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -21,7 +21,6 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben { ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(MicrobenchmarkAnalyzeCommand)); MicrobenchmarkConfiguration configuration = MicrobenchmarkConfigurationParser.Parse(settings.ConfigurationPath); - IReadOnlyList comparisonResults = MicrobenchmarkResultsAnalyzer.GetComparisons(configuration); Presentation.Present(configuration, new()); // Execution details aren't available for the analysis-only mode. return 0; } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index 22ddf41e6c5..94aaa8f29f1 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -121,7 +121,7 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi // Should only be one if it's a fresh run. string jsonFile = jsonFiles.First(); - MicrobenchmarkResults output = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); + BdnJsonResult output = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); // Assumption: A particular run, regardless of the parameters, will run ~the same vals. IEnumerable operationsPerNanos = output.Benchmarks.First().Measurements.Where(m => m.IterationMode == "Workload" && m.IterationStage == "Actual") diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt index 1bef991eca5..6056a312aff 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt @@ -11,7 +11,6 @@ "System.Tests.Perf_GC.NewOperator_Array(length: 10000)" | "System.Tests.Perf_GC.NewOperator_Array(length: 1000)" | "System.Tests.Perf_GC.NewOperator_Array(length: 10000)" | -"System.IO.Tests.Perf_File.ReadAllBytesAsync(size: 104857600)" | "System.Numerics.Tests.Perf_BigInteger.Subtract(arguments: 65536*" | "System.Collections.CtorGivenSize.Array(size: 512)" | "ByteMark.BenchBitOps" | From e34745beb3344cc9ce4e569554f24e2fdc23440b Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 1 May 2026 15:23:16 +0800 Subject: [PATCH 04/54] present list of microbenchmarkresults --- .../Analysis/GCTraceMetricComparisonResult.cs | 30 +++--- .../MicrobenchmarkComparisonResult.cs | 98 +++++++++++-------- .../MicrobenchmarkResultComparison.cs | 48 +++++++-- .../Presentation/Microbenchmarks/Json.cs | 8 +- .../Microbenchmarks/Presentation.cs | 11 +-- .../MicrobenchmarkAnalyzeCommand.cs | 27 ++++- .../Microbenchmark/MicrobenchmarkCommand.cs | 32 ++++-- 7 files changed, 160 insertions(+), 94 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs index 2ca6aadf149..0f6d69eb604 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs @@ -8,8 +8,8 @@ public sealed class GCTraceMetricComparisonResult { public GCTraceMetricComparisonResult(IEnumerable baselines, IEnumerable comparands, string metricName) { - Baselines = baselines; - Comparands = comparands; + RunName = baselines.FirstOrDefault()?.RunName; + Key = $"{baselines.FirstOrDefault()?.ConfigurationName}_{RunName}"; MetricName = metricName; PropertyInfo pInfo = typeof(GCTraceMetrics).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public); @@ -17,8 +17,8 @@ public GCTraceMetricComparisonResult(IEnumerable baselines, IEnu // Property found on the GCTraceMetrics. if (pInfo != null) { - OriginalBaselineMetricCollection = GoodLinq.Select(Baselines, baseline => (double)pInfo.GetValue(baseline)); - OriginalComparandMetricCollection = GoodLinq.Select(Comparands, comparand => (double)pInfo.GetValue(comparand)); + OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => (double)pInfo.GetValue(baseline)); + OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => (double)pInfo.GetValue(comparand)); } // If property isn't found on the GCTraceMetrics, look in GCStats. @@ -43,15 +43,15 @@ public GCTraceMetricComparisonResult(IEnumerable baselines, IEnu else { - OriginalBaselineMetricCollection = GoodLinq.Select(Baselines, baseline => (double)fieldInfo.GetValue(baseline)); - OriginalComparandMetricCollection = GoodLinq.Select(Comparands, comparand => (double)fieldInfo.GetValue(comparand)); + OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => (double)fieldInfo.GetValue(baseline)); + OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => (double)fieldInfo.GetValue(comparand)); } } else { - OriginalBaselineMetricCollection = GoodLinq.Select(Baselines, baseline => (double)pInfo.GetValue(baseline)); - OriginalComparandMetricCollection = GoodLinq.Select(Comparands, comparand => (double)pInfo.GetValue(comparand)); + OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => (double)pInfo.GetValue(baseline)); + OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => (double)pInfo.GetValue(comparand)); } } @@ -60,15 +60,12 @@ public GCTraceMetricComparisonResult(IEnumerable baselines, IEnu OutliersFreeComparandMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(OriginalComparandMetricCollection); // Calculate averaged metrics - AveragedBaselineMetric = OutliersFreeBaselineMetricCollection.Any() - ? OutliersFreeBaselineMetricCollection.Average() - : double.NaN; - AveragedComparandMetric = OutliersFreeComparandMetricCollection.Any() - ? OutliersFreeComparandMetricCollection.Average() - : double.NaN; + AveragedBaselineMetric = GoodLinq.Average(OutliersFreeBaselineMetricCollection, r => r); + AveragedComparandMetric = GoodLinq.Average(OutliersFreeComparandMetricCollection, r => r); } - public string RunName => Baselines.First().RunName; + public string RunName { get; } + public string Key { get; } public string MetricName { get; } public IEnumerable OriginalBaselineMetricCollection { get; } public IEnumerable OriginalComparandMetricCollection { get; } @@ -99,8 +96,5 @@ public double PercentageDelta } } } - public string Key => $"{Baselines.FirstOrDefault()?.ConfigurationName}_{RunName}"; - public IEnumerable Baselines { get; } - public IEnumerable Comparands { get; } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index eda6072ad80..8da35ef9133 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -9,10 +9,8 @@ public MicrobenchmarkComparisonResult() { } public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands) { - Baselines = baselines; - Comparands = comparands; - var baselineGCTraceMetricsCollection = GoodLinq.Select(Baselines, baseline => baseline.GCTraceMetrics); - var comparandGCTraceMetricsCollection = GoodLinq.Select(Comparands, comparand => comparand.GCTraceMetrics); + var baselineGCTraceMetricsCollection = GoodLinq.Select(baselines, baseline => baseline.GCTraceMetrics); + var comparandGCTraceMetricsCollection = GoodLinq.Select(comparands, comparand => comparand.GCTraceMetrics); string[] metricNames = new string[] { @@ -30,21 +28,20 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline GCTraceMetricComparison.CompareGCTraceMetric(baselineGCTraceMetricsCollection, comparandGCTraceMetricsCollection, metricName)); } - BaselineRunName = Baselines?.FirstOrDefault()?.Parent?.Name; - ComparandRunName = Comparands?.FirstOrDefault()?.Parent?.Name; - MicrobenchmarkName = Baselines?.FirstOrDefault()?.MicrobenchmarkName; + BaselineRunName = baselines?.FirstOrDefault()?.Parent?.Name; + ComparandRunName = comparands?.FirstOrDefault()?.Parent?.Name; + MicrobenchmarkName = baselines?.FirstOrDefault()?.MicrobenchmarkName; OriginalBaselineMeanValueCollection = - GoodLinq.Select(Baselines, baseline => baseline.Statistics?.Mean ?? double.NaN).ToArray(); + GoodLinq.Select(baselines, baseline => baseline.Statistics?.Mean ?? double.NaN).ToArray(); OriginalComparandMeanValueCollection = - GoodLinq.Select(Comparands, comparand => comparand.Statistics?.Mean ?? double.NaN).ToArray(); + GoodLinq.Select(comparands, comparand => comparand.Statistics?.Mean ?? double.NaN).ToArray(); } - public IEnumerable Baselines { get; set; } - public IEnumerable Comparands { get; set; } public List ComparisonResults { get; set; } public string BaselineRunName { get; } public string ComparandRunName { get; } + public string ComparisonName => $"{ComparandRunName} vs {BaselineRunName}"; public string MicrobenchmarkName { get; } public double[] OriginalBaselineMeanValueCollection { get; } public double[] OriginalComparandMeanValueCollection { get; } @@ -54,47 +51,66 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline public double[] OutliersFreeComparandMeanValueCollection => GC.Analysis.API.Statistics.RemoveOutliers(OriginalComparandMeanValueCollection).ToArray(); - public double MeanDiff => OutliersFreeComparandMeanValueCollection.Average() - OutliersFreeBaselineMeanValueCollection.Average(); - public double MeanDiffPerc => (MeanDiff / (Baselines.FirstOrDefault()?.Statistics?.Mean ?? double.NaN)) * 100; + public double AveragedBaselineMeanValue => GoodLinq.Average(OutliersFreeBaselineMeanValueCollection, r => r); + public double AveragedComparandMeanValue => GoodLinq.Average(OutliersFreeComparandMeanValueCollection, r => r); - public double? GetDiffPercentFromOtherMetrics(string metricName) - { - List baselineOtherMetricCollection = new(); - List comparandOtherMetricCollection = new(); - - foreach (var baseline in Baselines) + public double MeanDiff => AveragedComparandMeanValue - AveragedBaselineMeanValue; + public double MeanDiffPerc{ + get { - if (baseline.OtherMetrics.TryGetValue(metricName, out var baselineMetric)) + if (AveragedBaselineMeanValue == 0) { - if (baselineMetric.HasValue) + if (AveragedComparandMeanValue == 0) { - baselineOtherMetricCollection.Add(baselineMetric.Value); + return 0; } - } - } - - foreach (var comparand in Comparands) - { - if (comparand.OtherMetrics.TryGetValue(metricName, out var comparandMetric)) - { - if (comparandMetric.HasValue) + else { - comparandOtherMetricCollection.Add(comparandMetric.Value); + return double.NaN; } } + return (MeanDiff / AveragedBaselineMeanValue) * 100; } + } - if (baselineOtherMetricCollection.Count() * comparandOtherMetricCollection.Count() == 0) - { - return null; - } + //public double? GetDiffPercentFromOtherMetrics(string metricName) + //{ + // List baselineOtherMetricCollection = new(); + // List comparandOtherMetricCollection = new(); - var outliersFreeBaselineOtherMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(baselineOtherMetricCollection); - var outliersFreeComparandOtherMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(comparandOtherMetricCollection); + // foreach (var baseline in Baselines) + // { + // if (baseline.OtherMetrics.TryGetValue(metricName, out var baselineMetric)) + // { + // if (baselineMetric.HasValue) + // { + // baselineOtherMetricCollection.Add(baselineMetric.Value); + // } + // } + // } - var averagedOutliersFreeBaselineOtherMetric = outliersFreeBaselineOtherMetricCollection.Average(); - var averagedOutliersFreeComparandOtherMetric = outliersFreeComparandOtherMetricCollection.Average(); - return (averagedOutliersFreeBaselineOtherMetric - averagedOutliersFreeComparandOtherMetric) / averagedOutliersFreeBaselineOtherMetric; - } + // foreach (var comparand in Comparands) + // { + // if (comparand.OtherMetrics.TryGetValue(metricName, out var comparandMetric)) + // { + // if (comparandMetric.HasValue) + // { + // comparandOtherMetricCollection.Add(comparandMetric.Value); + // } + // } + // } + + // if (baselineOtherMetricCollection.Count() * comparandOtherMetricCollection.Count() == 0) + // { + // return null; + // } + + // var outliersFreeBaselineOtherMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(baselineOtherMetricCollection); + // var outliersFreeComparandOtherMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(comparandOtherMetricCollection); + + // var averagedOutliersFreeBaselineOtherMetric = outliersFreeBaselineOtherMetricCollection.Average(); + // var averagedOutliersFreeComparandOtherMetric = outliersFreeComparandOtherMetricCollection.Average(); + // return (averagedOutliersFreeBaselineOtherMetric - averagedOutliersFreeComparandOtherMetric) / averagedOutliersFreeBaselineOtherMetric; + //} } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 0dfe503b26a..f292bece3c0 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -40,6 +40,16 @@ public static class MicrobenchmarkResultComparison { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_10000_"}, }; + public static readonly Dictionary> DiffLevelPredicatorMap = new() + { + { "Large Regressions (>=20%)", r => r.MeanDiffPerc >= 20 }, + { "Large Improvements (<=-20%)", r => r.MeanDiffPerc <= -20 }, + { "Regressions (5% - 20%)", r => r.MeanDiffPerc >= 5 && r.MeanDiffPerc < 20 }, + { "Improvements (-20% - -5%)", r => r.MeanDiffPerc <= -5 && r.MeanDiffPerc > -20 }, + { "Stale Regressions (0% - 5%)", r => r.MeanDiffPerc >= 0 && r.MeanDiffPerc < 5 }, + { "Stale Improvements (0% - -5%)", r => r.MeanDiffPerc <= 0 && r.MeanDiffPerc > -5 }, + }; + private static readonly ConcurrentDictionary>> _benchmarkFullNameToJsonForRun = new(); public static ConcurrentDictionary> MapBenchmarkFullNameToJsonForRun(string outputPathForRun) @@ -50,7 +60,8 @@ public static ConcurrentDictionary> MapBenchmarkFullNameToJ string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); - Parallel.ForEach(jsonFiles, (jsonFile) => { + Parallel.ForEach(jsonFiles, (jsonFile) => + { BdnJsonResult results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); string fullName = results.Benchmarks.FirstOrDefault()?.FullName; benchmarkFullNameJsonMap[fullName] = benchmarkFullNameJsonMap.GetValueOrDefault(fullName, new()); @@ -69,7 +80,8 @@ public static ConcurrentDictionary MapJsonToTraceForSingleBenchm string[] jsonFiles = benchmarkFullNameJsonMap.GetValueOrDefault(benchmarkFullName, new()).ToArray(); - Parallel.ForEach(jsonFiles, (jsonFile) => { + Parallel.ForEach(jsonFiles, (jsonFile) => + { BdnJsonResult results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); string fullName = results.Benchmarks.FirstOrDefault()?.FullName; if (fullName != benchmarkFullName) @@ -86,7 +98,7 @@ public static ConcurrentDictionary MapJsonToTraceForSingleBenchm string traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkFullName]; - string[] sortedTraceFiles = Enumerable.Where(Directory.GetFiles(outputPathForRun, "*.etlx", SearchOption.TopDirectoryOnly), traceFile => + string[] sortedTraceFiles = Enumerable.Where(Directory.GetFiles(outputPathForRun, "*.etl.zip", SearchOption.TopDirectoryOnly), traceFile => Path.GetFileName(traceFile).ToLower().Contains(traceFileNameTemplate.ToLower())) .OrderBy(traceFile => traceFile) .ToArray(); @@ -105,7 +117,7 @@ public static ConcurrentDictionary MapJsonToTraceForSingleBenchm return jsonTraceMap; } - public static IReadOnlyDictionary> AnalyzeMicrobenchmarkResultsForSingleBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) + public static IReadOnlyDictionary> AnalyzeMicrobenchmarkResultsForSingleBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) { ConcurrentDictionary> runsToResults = new(); @@ -118,7 +130,8 @@ public static IReadOnlyDictionary> AnalyzeMicrobe runsToResults[run.Value] = runsToResults.GetValueOrDefault(run.Value, new()); - Parallel.ForEach(jsonTraceMap, jsonTracePair => { + Parallel.ForEach(jsonTraceMap, jsonTracePair => + { string jsonPath = jsonTracePair.Key; string tracePath = jsonTracePair.Value; @@ -137,8 +150,7 @@ public static IReadOnlyDictionary> AnalyzeMicrobe if (!excludeTraces) { - Analyzer analyzer = AnalyzerManager.GetAnalyzer(tracePath); - + using var analyzer = AnalyzerManager.GetAnalyzer(tracePath); List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); List corerunProcesses = analyzer.GetProcessGCData("corerun"); allPertinentProcesses.AddRange(corerunProcesses); @@ -178,7 +190,7 @@ public static IReadOnlyDictionary> AnalyzeMicrobe runsToResults[run.Value].Add(microbenchmarkResult); } }); - + }); return runsToResults; @@ -188,7 +200,6 @@ public static List CompareMicrobenchmarkResultFo { IReadOnlyDictionary> runResults = AnalyzeMicrobenchmarkResultsForSingleBenchmark(configuration, benchmarkFullName, excludeTraces); List comparisonResults = new(); - if (configuration.Output.run_comparisons != null) { foreach (var comparison in configuration.Output.run_comparisons) @@ -221,5 +232,22 @@ public static List CompareMicrobenchmarkResultFo return comparisonResults; } + + public static List GroupComparisonResultsByName(MicrobenchmarkConfiguration configuration, List comparisonResultForAllBenchmarks, bool excludeTraces = false) + { + List allComparisonResults = new(); + + comparisonResultForAllBenchmarks + .GroupBy(r => r.ComparisonName) + .ToList() + .ForEach(group => + { + string baselineName = group.FirstOrDefault()?.BaselineRunName ?? "Baseline"; + string runName = group.FirstOrDefault()?.ComparandRunName ?? "Comparand"; + allComparisonResults.Add(new(baselineName, runName, group.ToList())); + }); + + return allComparisonResults; + } } -} +} \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs index 3b2e4eeef4f..9bf2736ef0a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs @@ -1,14 +1,16 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Analysis.API; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using GC.Infrastructure.Core.Presentation.GCPerfSim; using Newtonsoft.Json; namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Json { - public static void Generate(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResults, string path) + public static void Generate(MicrobenchmarkConfiguration configuration, List comparisonResultsGroupedByName, string path) { - string json = JsonConvert.SerializeObject(comparisonResults); + string json = JsonConvert.SerializeObject(comparisonResultsGroupedByName); File.WriteAllText(path, json); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs index 6a3a426c532..cb0fd4db214 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs @@ -6,27 +6,22 @@ namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Presentation { - public static IReadOnlyList Present(MicrobenchmarkConfiguration configuration, Dictionary executionDetails) + public static void Present(MicrobenchmarkConfiguration configuration, List comparisonResultsGroupedByName, Dictionary executionDetails) { - //IReadOnlyList comparisonResults = MicrobenchmarkResultComparison.CompareMicrobenchmarkResults(configuration); - IReadOnlyList comparisonResults = Array.Empty().ToList(); - foreach (var format in configuration.Output.Formats) { if (format == "markdown") { - //Markdown.GenerateTable(configuration, comparisonResults, executionDetails, Path.Combine(configuration.Output.Path, "Results.md")); + //Markdown.GenerateTable(configuration, comparisonResultsGroupedByName, executionDetails, Path.Combine(configuration.Output.Path, "Results.md")); continue; } if (format == "json") { - //Json.Json.Generate(configuration, comparisonResults, Path.Combine(configuration.Output.Path, "Results.json")); + Json.Generate(configuration, comparisonResultsGroupedByName, Path.Combine(configuration.Output.Path, "Results.json")); continue; } } - - return comparisonResults; } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index 976e75c07aa..48f45efb872 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -1,10 +1,13 @@ -using Spectre.Console.Cli; -using System.ComponentModel; -using System.Diagnostics.CodeAnalysis; +using GC.Analysis.API; using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Infrastructure.Core.Presentation.Microbenchmarks; using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using GC.Infrastructure.Core.Presentation.GCPerfSim; +using GC.Infrastructure.Core.Presentation.Microbenchmarks; +using Spectre.Console.Cli; +using System.Collections.Concurrent; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; namespace GC.Infrastructure.Commands.Microbenchmark { @@ -21,7 +24,21 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben { ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(MicrobenchmarkAnalyzeCommand)); MicrobenchmarkConfiguration configuration = MicrobenchmarkConfigurationParser.Parse(settings.ConfigurationPath); - Presentation.Present(configuration, new()); // Execution details aren't available for the analysis-only mode. + + Run run = configuration.Runs.Values.FirstOrDefault(); + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name); + var benchmarkFullNameJsonMap = MicrobenchmarkResultComparison.MapBenchmarkFullNameToJsonForRun(outputPathForRun); + List comparisonResultForAllBenchmarks = new(); + + foreach (var benchmarkFullName in benchmarkFullNameJsonMap.Keys) + { + List comparisonResultsForBenchmark = MicrobenchmarkResultComparison.CompareMicrobenchmarkResultForBenchmark(configuration, benchmarkFullName); + comparisonResultForAllBenchmarks.AddRange(comparisonResultsForBenchmark); + } + + var comparisonResultsGroupedName = MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResultForAllBenchmarks); + + Presentation.Present(configuration, comparisonResultsGroupedName, new()); // Execution details aren't available for the analysis-only mode. return 0; } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index 94aaa8f29f1..d84f9981895 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -1,18 +1,19 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Analysis.API; using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.CommandBuilders; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.Configurations; +using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.Presentation.Microbenchmarks; using GC.Infrastructure.Core.TraceCollection; using Newtonsoft.Json; -using Spectre.Console.Cli; using Spectre.Console; +using Spectre.Console.Cli; using System.ComponentModel; -using System.Diagnostics.CodeAnalysis; +using System.Configuration; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Text; -using System.Configuration; namespace GC.Infrastructure.Commands.Microbenchmark { @@ -124,8 +125,8 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi BdnJsonResult output = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); // Assumption: A particular run, regardless of the parameters, will run ~the same vals. - IEnumerable operationsPerNanos = output.Benchmarks.First().Measurements.Where(m => m.IterationMode == "Workload" && m.IterationStage == "Actual") - .Select(m => m.Operations); + var operationsPerNanos = GoodLinq.Select(GoodLinq.Where(output.Benchmarks.First().Measurements, m => m.IterationMode == "Workload" && m.IterationStage == "Actual"), m => m.Operations); + // For now take the max but we will possibly be sacrificing duration for precision. invocationCountFromBaseline = operationsPerNanos.Max(); } @@ -199,10 +200,23 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi } } - IReadOnlyList results = Presentation.Present(configuration, executionDetails); + Run sampleRun = configuration.Runs.Values.FirstOrDefault(); + string outputPathForRun = Path.Combine(configuration.Output.Path, sampleRun.Name); + var benchmarkFullNameJsonMap = MicrobenchmarkResultComparison.MapBenchmarkFullNameToJsonForRun(outputPathForRun); + List comparisonResultForAllBenchmarks = new(); + + foreach (string benchmarkFullName in benchmarkFullNameJsonMap.Keys) + { + List comparisonResultsForBenchmark = MicrobenchmarkResultComparison.CompareMicrobenchmarkResultForBenchmark(configuration, benchmarkFullName); + comparisonResultForAllBenchmarks.AddRange(comparisonResultsForBenchmark); + } + + var comparisonResultsGroupedName = MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResultForAllBenchmarks); + + Presentation.Present(configuration, comparisonResultsGroupedName, new()); // Execution details aren't available for the analysis-only mode. Directory.SetCurrentDirectory(currentDirectory); AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbechmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); - return new MicrobenchmarkOutputResults(executionDetails, results); + return new MicrobenchmarkOutputResults(executionDetails, comparisonResultsGroupedName); } } } From e9594b330fbb12d99293e0f5f00578472fa475a1 Mon Sep 17 00:00:00 2001 From: VincentBu <44959937+VincentBu@users.noreply.github.com> Date: Wed, 6 May 2026 13:24:59 +0800 Subject: [PATCH 05/54] rename iteration section to iterations for Run.yaml Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml b/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml index 33512caec9f..17be7e9ab35 100644 --- a/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml @@ -12,7 +12,7 @@ coreruns: environment_variables: DOTNET_GCName: clrgc.dll -iteration: +iterations: gcperfsim: 1 microbenchmarks: 1 From c3f4b4594c0b8d97c4f6354d4ec968ee22d13d99 Mon Sep 17 00:00:00 2001 From: VincentBu <44959937+VincentBu@users.noreply.github.com> Date: Wed, 6 May 2026 14:26:30 +0800 Subject: [PATCH 06/54] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../Configurations/Microbenchmarks.Configuration.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs index 7ea7f6ac4bd..ab9ab01a911 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs @@ -25,6 +25,12 @@ public class Environment { public uint default_max_seconds { get; set; } = 300; public uint iterations { get; set; } = 1; + + [YamlMember(Alias = "iteration")] + public uint iteration + { + set => iterations = value; + } } public sealed class MicrobenchmarkConfigurations From 240ceb8caebec9d8a01960b4ad4a07e647e446cf Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Wed, 6 May 2026 14:41:56 +0800 Subject: [PATCH 07/54] fix for microbencharmks comparison --- .../Analysis/GCTraceMetrics.cs | 4 +- .../Microbenchmarks/MicrobenchmarkResult.cs | 8 ++-- .../MicrobenchmarkResultComparison.cs | 43 +++++++++++++------ .../Microbenchmark/MicrobenchmarkCommand.cs | 2 +- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs index fb0deabc532..e6133161c83 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs @@ -61,7 +61,7 @@ public GCTraceMetrics(GCProcessData processData, string runName, string configur var properties = processData.Stats.GetType().GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); foreach (var property in properties) { - if (property.PropertyType != typeof(double) || property.PropertyType != typeof(int)) + if (property.PropertyType != typeof(double) && property.PropertyType != typeof(int)) { continue; } @@ -74,7 +74,7 @@ public GCTraceMetrics(GCProcessData processData, string runName, string configur var fields = processData.Stats.GetType().GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); foreach (var field in fields) { - if (field.FieldType != typeof(double) || field.FieldType != typeof(int)) + if (field.FieldType != typeof(double) && field.FieldType != typeof(int)) { continue; } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs index 9c24753d7c9..62fff5e0c0b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -6,17 +6,17 @@ namespace GC.Infrastructure.Core.Analysis { public sealed class MicrobenchmarkResult { - public Statistics Statistics { get; set; } + public Statistics? Statistics { get; set; } [JsonIgnore] public GCProcessData? GCData { get; set; } - public GCTraceMetrics GCTraceMetrics { get; set; } + public GCTraceMetrics? GCTraceMetrics { get; set; } [JsonIgnore] public CPUProcessData? CPUData { get; set; } - public Run Parent { get; set; } - public string MicrobenchmarkName { get; set; } + public Run? Parent { get; set; } + public string? MicrobenchmarkName { get; set; } public Dictionary OtherMetrics { get; set; } = new(); private static readonly IReadOnlyDictionary> _customStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index f292bece3c0..91a5e7c6c27 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -50,22 +50,25 @@ public static class MicrobenchmarkResultComparison { "Stale Improvements (0% - -5%)", r => r.MeanDiffPerc <= 0 && r.MeanDiffPerc > -5 }, }; - private static readonly ConcurrentDictionary>> _benchmarkFullNameToJsonForRun = new(); + private static readonly ConcurrentDictionary>> _benchmarkFullNameToJsonForRun = new(); - public static ConcurrentDictionary> MapBenchmarkFullNameToJsonForRun(string outputPathForRun) + public static ConcurrentDictionary> MapBenchmarkFullNameToJsonForRun(string outputPathForRun) { return _benchmarkFullNameToJsonForRun.GetOrAdd(outputPathForRun, path => { - ConcurrentDictionary> benchmarkFullNameJsonMap = new(); + ConcurrentDictionary> benchmarkFullNameJsonMap = new(); string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); Parallel.ForEach(jsonFiles, (jsonFile) => { - BdnJsonResult results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); - string fullName = results.Benchmarks.FirstOrDefault()?.FullName; - benchmarkFullNameJsonMap[fullName] = benchmarkFullNameJsonMap.GetValueOrDefault(fullName, new()); - benchmarkFullNameJsonMap[fullName].Add(jsonFile); + BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); + string? fullName = results?.Benchmarks?.FirstOrDefault()?.FullName; + if (fullName != null) + { + benchmarkFullNameJsonMap[fullName] = benchmarkFullNameJsonMap.GetValueOrDefault(fullName, new()); + benchmarkFullNameJsonMap[fullName].Add(jsonFile); + } }); return benchmarkFullNameJsonMap; @@ -82,8 +85,8 @@ public static ConcurrentDictionary MapJsonToTraceForSingleBenchm Parallel.ForEach(jsonFiles, (jsonFile) => { - BdnJsonResult results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); - string fullName = results.Benchmarks.FirstOrDefault()?.FullName; + BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); + string? fullName = results?.Benchmarks?.FirstOrDefault()?.FullName; if (fullName != benchmarkFullName) { return; @@ -96,6 +99,10 @@ public static ConcurrentDictionary MapJsonToTraceForSingleBenchm .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) .ToArray(); + if (!_benchmarkNameToTraceFilePatternMap.Keys.Contains(benchmarkFullName)) + { + throw new KeyNotFoundException("No trace file pattern found for benchmark: " + benchmarkFullName); + } string traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkFullName]; string[] sortedTraceFiles = Enumerable.Where(Directory.GetFiles(outputPathForRun, "*.etl.zip", SearchOption.TopDirectoryOnly), traceFile => @@ -117,9 +124,9 @@ public static ConcurrentDictionary MapJsonToTraceForSingleBenchm return jsonTraceMap; } - public static IReadOnlyDictionary> AnalyzeMicrobenchmarkResultsForSingleBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) + public static IReadOnlyDictionary> AnalyzeMicrobenchmarkResultsForSingleBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) { - ConcurrentDictionary> runsToResults = new(); + ConcurrentDictionary> runsToResults = new(); Parallel.ForEach(configuration.Runs, (run) => { @@ -135,9 +142,16 @@ public static IReadOnlyDictionary> AnalyzeMicrob string jsonPath = jsonTracePair.Key; string tracePath = jsonTracePair.Value; - BdnJsonResult results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); + BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); + + List? benchmarks = results?.Benchmarks; + + if (benchmarks == null) + { + return; + } - foreach (var benchmark in results?.Benchmarks) + foreach (var benchmark in benchmarks) { Statistics statistics = benchmark.Statistics; @@ -146,6 +160,7 @@ public static IReadOnlyDictionary> AnalyzeMicrob Statistics = statistics, Parent = run.Value, MicrobenchmarkName = benchmarkFullName, + GCTraceMetrics = GCTraceMetrics.GetNullItem(tracePath, benchmark.FullName) }; if (!excludeTraces) @@ -198,7 +213,7 @@ public static IReadOnlyDictionary> AnalyzeMicrob public static List CompareMicrobenchmarkResultForBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) { - IReadOnlyDictionary> runResults = AnalyzeMicrobenchmarkResultsForSingleBenchmark(configuration, benchmarkFullName, excludeTraces); + IReadOnlyDictionary> runResults = AnalyzeMicrobenchmarkResultsForSingleBenchmark(configuration, benchmarkFullName, excludeTraces); List comparisonResults = new(); if (configuration.Output.run_comparisons != null) { diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index d84f9981895..1b43a835f73 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -213,7 +213,7 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi var comparisonResultsGroupedName = MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResultForAllBenchmarks); - Presentation.Present(configuration, comparisonResultsGroupedName, new()); // Execution details aren't available for the analysis-only mode. + Presentation.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. Directory.SetCurrentDirectory(currentDirectory); AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbechmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); return new MicrobenchmarkOutputResults(executionDetails, comparisonResultsGroupedName); From 1b08bd0fcc0efb87dca14368996040400eeeb26c Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Wed, 6 May 2026 17:53:24 +0800 Subject: [PATCH 08/54] fix bugs intrduced in previous commit --- .../GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs | 4 ++-- .../Commands/Microbenchmark/MicrobenchmarkCommand.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs index e6133161c83..fb0deabc532 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs @@ -61,7 +61,7 @@ public GCTraceMetrics(GCProcessData processData, string runName, string configur var properties = processData.Stats.GetType().GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); foreach (var property in properties) { - if (property.PropertyType != typeof(double) && property.PropertyType != typeof(int)) + if (property.PropertyType != typeof(double) || property.PropertyType != typeof(int)) { continue; } @@ -74,7 +74,7 @@ public GCTraceMetrics(GCProcessData processData, string runName, string configur var fields = processData.Stats.GetType().GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); foreach (var field in fields) { - if (field.FieldType != typeof(double) && field.FieldType != typeof(int)) + if (field.FieldType != typeof(double) || field.FieldType != typeof(int)) { continue; } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index 1b43a835f73..4f962dc0c31 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -152,7 +152,7 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi for (int index = 0; index < configuration.Environment.iterations; index++) { - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbechmarks: {configuration.Name} - {run.Key} {benchmark} - iteration: {index} [/]\n"); + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbenchmarks: {configuration.Name} - {run.Key} {benchmark} - iteration: {index} [/]\n"); // Run The BDN process with the trace collector. using (Process bdnProcess = new()) { From 434a17c5c0eed8172535b4ecacf129f634663ae1 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 7 May 2026 10:06:48 +0800 Subject: [PATCH 09/54] Add json-only comparison --- .../MicrobenchmarkComparisonResult.cs | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 8da35ef9133..9b4f2dddcd6 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -7,25 +7,28 @@ public sealed class MicrobenchmarkComparisonResult { public MicrobenchmarkComparisonResult() { } - public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands) + public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands, bool excludeTraces = false) { - var baselineGCTraceMetricsCollection = GoodLinq.Select(baselines, baseline => baseline.GCTraceMetrics); - var comparandGCTraceMetricsCollection = GoodLinq.Select(comparands, comparand => comparand.GCTraceMetrics); - - string[] metricNames = new string[] + ComparisonResults = new(); + if (!excludeTraces) { + var baselineGCTraceMetricsCollection = GoodLinq.Select(baselines, baseline => baseline.GCTraceMetrics); + var comparandGCTraceMetricsCollection = GoodLinq.Select(comparands, comparand => comparand.GCTraceMetrics); + + string[] metricNames = new string[] + { "PctTimePausedInGC", "ExecutionTimeMSec", "PauseDurationMSec_MeanWhereIsEphemeral", "PauseDurationMSec_MeanWhereIsBackground", "PauseDurationMSec_MeanWhereIsBlockingGen2" - }; + }; - ComparisonResults = new(); - foreach (var metricName in metricNames) - { - ComparisonResults.Add( - GCTraceMetricComparison.CompareGCTraceMetric(baselineGCTraceMetricsCollection, comparandGCTraceMetricsCollection, metricName)); + foreach (var metricName in metricNames) + { + ComparisonResults.Add( + GCTraceMetricComparison.CompareGCTraceMetric(baselineGCTraceMetricsCollection, comparandGCTraceMetricsCollection, metricName)); + } } BaselineRunName = baselines?.FirstOrDefault()?.Parent?.Name; From faca6283c287cafe33ee9909ddcdf738af36e713 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 7 May 2026 10:07:58 +0800 Subject: [PATCH 10/54] extract a shared helper for analyze command --- .../Microbenchmark/MicrobenchmarkAnalyzeCommand.cs | 13 +++++++++---- .../Microbenchmark/MicrobenchmarkCommand.cs | 13 +------------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index 48f45efb872..347d7d5f035 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -25,6 +25,14 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(MicrobenchmarkAnalyzeCommand)); MicrobenchmarkConfiguration configuration = MicrobenchmarkConfigurationParser.Parse(settings.ConfigurationPath); + var comparisonResultsGroupedName = ExecuteAnalysis(configuration); + + Presentation.Present(configuration, comparisonResultsGroupedName, new()); // Execution details aren't available for the analysis-only mode. + return 0; + } + + public static List ExecuteAnalysis(MicrobenchmarkConfiguration configuration) + { Run run = configuration.Runs.Values.FirstOrDefault(); string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name); var benchmarkFullNameJsonMap = MicrobenchmarkResultComparison.MapBenchmarkFullNameToJsonForRun(outputPathForRun); @@ -36,10 +44,7 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben comparisonResultForAllBenchmarks.AddRange(comparisonResultsForBenchmark); } - var comparisonResultsGroupedName = MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResultForAllBenchmarks); - - Presentation.Present(configuration, comparisonResultsGroupedName, new()); // Execution details aren't available for the analysis-only mode. - return 0; + return MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResultForAllBenchmarks); } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index 4f962dc0c31..74cb749e680 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -200,18 +200,7 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi } } - Run sampleRun = configuration.Runs.Values.FirstOrDefault(); - string outputPathForRun = Path.Combine(configuration.Output.Path, sampleRun.Name); - var benchmarkFullNameJsonMap = MicrobenchmarkResultComparison.MapBenchmarkFullNameToJsonForRun(outputPathForRun); - List comparisonResultForAllBenchmarks = new(); - - foreach (string benchmarkFullName in benchmarkFullNameJsonMap.Keys) - { - List comparisonResultsForBenchmark = MicrobenchmarkResultComparison.CompareMicrobenchmarkResultForBenchmark(configuration, benchmarkFullName); - comparisonResultForAllBenchmarks.AddRange(comparisonResultsForBenchmark); - } - - var comparisonResultsGroupedName = MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResultForAllBenchmarks); + var comparisonResultsGroupedName = MicrobenchmarkAnalyzeCommand.ExecuteAnalysis(configuration); Presentation.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. Directory.SetCurrentDirectory(currentDirectory); From ab17bf3278f1228a7fb158acc3d19a45be2ba37a Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 7 May 2026 16:36:21 +0800 Subject: [PATCH 11/54] take trace type into consideration --- .../MicrobenchmarkComparisonResult.cs | 4 +-- .../MicrobenchmarkResultComparison.cs | 29 ++++++++----------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 9b4f2dddcd6..36583ec46e8 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -7,10 +7,10 @@ public sealed class MicrobenchmarkComparisonResult { public MicrobenchmarkComparisonResult() { } - public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands, bool excludeTraces = false) + public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands, bool includeTraces = true) { ComparisonResults = new(); - if (!excludeTraces) + if (includeTraces) { var baselineGCTraceMetricsCollection = GoodLinq.Select(baselines, baseline => baseline.GCTraceMetrics); var comparandGCTraceMetricsCollection = GoodLinq.Select(comparands, comparand => comparand.GCTraceMetrics); diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 91a5e7c6c27..182b838a67f 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -85,12 +85,6 @@ public static ConcurrentDictionary MapJsonToTraceForSingleBenchm Parallel.ForEach(jsonFiles, (jsonFile) => { - BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); - string? fullName = results?.Benchmarks?.FirstOrDefault()?.FullName; - if (fullName != benchmarkFullName) - { - return; - } // placeholder jsonTraceMap[jsonFile] = ""; }); @@ -101,7 +95,7 @@ public static ConcurrentDictionary MapJsonToTraceForSingleBenchm if (!_benchmarkNameToTraceFilePatternMap.Keys.Contains(benchmarkFullName)) { - throw new KeyNotFoundException("No trace file pattern found for benchmark: " + benchmarkFullName); + throw new KeyNotFoundException("No trace file pattern found for benchmark: " + benchmarkFullName); } string traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkFullName]; @@ -133,15 +127,13 @@ public static IReadOnlyDictionary> Anal string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); run.Value.Name ??= run.Key; - var jsonTraceMap = MapJsonToTraceForSingleBenchmarkRun(outputPathForRun, benchmarkFullName); + var benchmarkToJsonMapForRun = MapBenchmarkFullNameToJsonForRun(outputPathForRun); + var jsonFiles = benchmarkToJsonMapForRun.GetValueOrDefault(benchmarkFullName, new()); runsToResults[run.Value] = runsToResults.GetValueOrDefault(run.Value, new()); - Parallel.ForEach(jsonTraceMap, jsonTracePair => + Parallel.ForEach(jsonFiles, jsonPath => { - string jsonPath = jsonTracePair.Key; - string tracePath = jsonTracePair.Value; - BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); List? benchmarks = results?.Benchmarks; @@ -159,12 +151,14 @@ public static IReadOnlyDictionary> Anal { Statistics = statistics, Parent = run.Value, - MicrobenchmarkName = benchmarkFullName, - GCTraceMetrics = GCTraceMetrics.GetNullItem(tracePath, benchmark.FullName) + MicrobenchmarkName = benchmarkFullName }; - if (!excludeTraces) + if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") { + var jsonTraceMap = MapJsonToTraceForSingleBenchmarkRun(outputPathForRun, benchmarkFullName); + string tracePath = jsonTraceMap.GetValueOrDefault(jsonPath, ""); + using var analyzer = AnalyzerManager.GetAnalyzer(tracePath); List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); List corerunProcesses = analyzer.GetProcessGCData("corerun"); @@ -213,6 +207,7 @@ public static IReadOnlyDictionary> Anal public static List CompareMicrobenchmarkResultForBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) { + bool includeTraces = (!excludeTraces) && configuration.TraceConfigurations.Type != "none"; IReadOnlyDictionary> runResults = AnalyzeMicrobenchmarkResultsForSingleBenchmark(configuration, benchmarkFullName, excludeTraces); List comparisonResults = new(); if (configuration.Output.run_comparisons != null) @@ -229,7 +224,7 @@ public static List CompareMicrobenchmarkResultFo var baselineMicrobenchmarkResults = GoodLinq.Select(baselineRuns, b => runResults[b]).SelectMany(r => r); var comparandMicrobenchmarkResults = GoodLinq.Select(comparandRuns, c => runResults[c]).SelectMany(r => r); - comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults)); + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); } } @@ -242,7 +237,7 @@ public static List CompareMicrobenchmarkResultFo var baselineMicrobenchmarkResults = GoodLinq.Select(baselineRuns, b => runResults[b]).SelectMany(r => r); var comparandMicrobenchmarkResults = GoodLinq.Select(comparandRuns, c => runResults[c]).SelectMany(r => r); - comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults)); + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); } return comparisonResults; From 70c7f167c29ac1337e195c22b5f371c8e1396bbc Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 7 May 2026 17:07:59 +0800 Subject: [PATCH 12/54] move MicrobenchmarkResult to GC.Infrastructure.Core.Analysis.Microbenchmarks namespace --- .../Analysis/Microbenchmarks/MicrobenchmarkResult.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs index 62fff5e0c0b..128d05be573 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -2,7 +2,7 @@ using GC.Infrastructure.Core.Configurations.Microbenchmarks; using Newtonsoft.Json; -namespace GC.Infrastructure.Core.Analysis +namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { public sealed class MicrobenchmarkResult { From d9b160df4c4865ced1b78ae3a8f16e7c5f7cc3fb Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 7 May 2026 17:08:23 +0800 Subject: [PATCH 13/54] validate if run is null --- .../Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index 347d7d5f035..9caa4db7889 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -33,7 +33,11 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben public static List ExecuteAnalysis(MicrobenchmarkConfiguration configuration) { - Run run = configuration.Runs.Values.FirstOrDefault(); + Run? run = configuration.Runs.Values.FirstOrDefault(); + if (run == null) + { + throw new InvalidOperationException("No runs found in the configuration."); + } string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name); var benchmarkFullNameJsonMap = MicrobenchmarkResultComparison.MapBenchmarkFullNameToJsonForRun(outputPathForRun); List comparisonResultForAllBenchmarks = new(); From 2ba9f6b87c0ed315238afaa545fb18ec051e67cc Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 7 May 2026 17:16:28 +0800 Subject: [PATCH 14/54] rename PauseDurationSeconds_SumWhereIsGen1 to PauseDurationMSec_SumWhereIsGen1 --- .../GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs index fb0deabc532..be56b376507 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs @@ -28,7 +28,7 @@ private GCTraceMetrics(string runName, string corerun) PauseDurationMSec_95PWhereIsBlockingGen2 = double.NaN; PauseDurationMSec_MeanWhereIsBlockingGen2 = double.NaN; CountIsBlockingGen2 = double.NaN; - PauseDurationSeconds_SumWhereIsGen1 = double.NaN; + PauseDurationMSec_SumWhereIsGen1 = double.NaN; PauseDurationMSec_MeanWhereIsEphemeral = double.NaN; PromotedMB_MeanWhereIsGen1 = double.NaN; CountIsGen1 = double.NaN; @@ -104,7 +104,7 @@ public GCTraceMetrics(GCProcessData processData, string runName, string configur PauseDurationMSec_MeanWhereIsEphemeral = GoodLinq.Average(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1 || gc.Generation == 0)), (gc => gc.PauseDurationMSec)); - PauseDurationSeconds_SumWhereIsGen1 = + PauseDurationMSec_SumWhereIsGen1 = GoodLinq.Sum(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PauseDurationMSec)); PauseDurationMSec_Sum = GoodLinq.Sum(processData.GCs, (gc => gc.PauseDurationMSec)); CountIsGen1 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 1).Count; @@ -127,7 +127,7 @@ public GCTraceMetrics(GCProcessData processData, string runName, string configur public double PauseDurationMSec_95PWhereIsBlockingGen2 { get; } public double PauseDurationMSec_MeanWhereIsBlockingGen2 { get; } public double CountIsBlockingGen2 { get; } - public double PauseDurationSeconds_SumWhereIsGen1 { get; } + public double PauseDurationMSec_SumWhereIsGen1 { get; } public double PauseDurationMSec_MeanWhereIsEphemeral { get; } public double PromotedMB_MeanWhereIsGen1 { get; } public double CountIsGen1 { get; } From 1dc65f53586bbb8a34df82498e580789f79dc28d Mon Sep 17 00:00:00 2001 From: VincentBu <44959937+VincentBu@users.noreply.github.com> Date: Thu, 7 May 2026 17:37:28 +0800 Subject: [PATCH 15/54] assign value for PromotedMB_MeanWhereIsGen1 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs index be56b376507..df14a2368f9 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs @@ -106,6 +106,8 @@ public GCTraceMetrics(GCProcessData processData, string runName, string configur GoodLinq.Average(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1 || gc.Generation == 0)), (gc => gc.PauseDurationMSec)); PauseDurationMSec_SumWhereIsGen1 = GoodLinq.Sum(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PauseDurationMSec)); + PromotedMB_MeanWhereIsGen1 = + GoodLinq.Average(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PromotedMB)); PauseDurationMSec_Sum = GoodLinq.Sum(processData.GCs, (gc => gc.PauseDurationMSec)); CountIsGen1 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 1).Count; CountIsGen0 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 0).Count; From 691500928053afed2fb4cc8ba84253abbb781b6f Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Sat, 9 May 2026 13:43:56 +0800 Subject: [PATCH 16/54] add properties for microbenchmarks comparison --- .../MicrobenchmarkComparisonResult.cs | 198 +++++++++++++----- .../Microbenchmarks/MicrobenchmarkResult.cs | 41 ---- .../Presentation/Microbenchmarks/Markdown.cs | 185 ++++++++-------- .../Microbenchmarks/Presentation.cs | 2 +- .../MicrobenchmarkAnalyzeCommand.cs | 5 +- 5 files changed, 240 insertions(+), 191 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 36583ec46e8..6ee5218dad5 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -1,10 +1,49 @@ using GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { // Per Microbenchmark result. public sealed class MicrobenchmarkComparisonResult { + public static readonly string[] RequiredMetrics = new string[] + { + "PctTimePausedInGC", + "ExecutionTimeMSec", + "PauseDurationMSec_MeanWhereIsEphemeral", + "PauseDurationMSec_MeanWhereIsBackground", + "PauseDurationMSec_MeanWhereIsBlockingGen2" + }; + + public static readonly IReadOnlyDictionary> CustomStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "number of iterations", (Statistics stats) => stats.N }, + { "min", (Statistics stats) => stats.Min }, + { "max", (Statistics stats) => stats.Max }, + { "median", (Statistics stats) => stats.Median }, + { "q1", (Statistics stats) => stats.Q1 }, + { "q3", (Statistics stats) => stats.Q3 }, + { "variance", (Statistics stats) => stats.Variance }, + { "standard deviation", (Statistics stats) => stats.StandardDeviation }, + { "skewness", (Statistics stats) => stats.Skewness }, + { "kurtosis", (Statistics stats) => stats.Kurtosis }, + { "standard error", (Statistics stats) => stats.StandardError }, + { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, + }; + + public static readonly Dictionary> CustomAggregateCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "gc count", (gc) => gc.Stats.Count }, + { "non induced gc count", (gc) => gc.Stats.Count - gc.GCs.Count(g => g.Reason == GCReason.Induced)}, + { "induced gc count", (gc) => gc.GCs.Count(g => g.Reason == GCReason.Induced)}, + { "total allocated (mb)", (gc) => gc.Stats.TotalAllocatedMB }, + { "max size peak (mb)", (gc) => gc.Stats.MaxSizePeakMB }, + { "total pause time (msec)", (gc) => gc.Stats.TotalPauseTimeMSec }, + { "gc pause time %", (gc) => gc.Stats.GetGCPauseTimePercentage() }, + { "avg. heap size (mb)", (gc) => GoodLinq.Average(gc.GCs, g => g.HeapSizeBeforeMB) }, + { "avg. heap size after (mb)", (gc) => GoodLinq.Average(gc.GCs, g => g.HeapSizeAfterMB) }, + }; + public MicrobenchmarkComparisonResult() { } public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands, bool includeTraces = true) @@ -15,20 +54,24 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline var baselineGCTraceMetricsCollection = GoodLinq.Select(baselines, baseline => baseline.GCTraceMetrics); var comparandGCTraceMetricsCollection = GoodLinq.Select(comparands, comparand => comparand.GCTraceMetrics); - string[] metricNames = new string[] - { - "PctTimePausedInGC", - "ExecutionTimeMSec", - "PauseDurationMSec_MeanWhereIsEphemeral", - "PauseDurationMSec_MeanWhereIsBackground", - "PauseDurationMSec_MeanWhereIsBlockingGen2" - }; - - foreach (var metricName in metricNames) + foreach (var metricName in RequiredMetrics) { ComparisonResults.Add( GCTraceMetricComparison.CompareGCTraceMetric(baselineGCTraceMetricsCollection, comparandGCTraceMetricsCollection, metricName)); } + + foreach (var kvp in CustomAggregateCalculationMap) + { + string customGCData = kvp.Key; + Func calculation = kvp.Value; + OriginalBaselineCustomGCData[customGCData] = Array.Empty(); + OriginalComparandCustomGCData[customGCData] = Array.Empty(); + + OriginalBaselineCustomGCData[customGCData] = + GoodLinq.Select(baselines, baseline => calculation(baseline.GCData)).ToArray(); + OriginalComparandCustomGCData[customGCData] = + GoodLinq.Select(comparands, comparand => calculation(comparand.GCData)).ToArray(); + } } BaselineRunName = baselines?.FirstOrDefault()?.Parent?.Name; @@ -39,6 +82,19 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline GoodLinq.Select(baselines, baseline => baseline.Statistics?.Mean ?? double.NaN).ToArray(); OriginalComparandMeanValueCollection = GoodLinq.Select(comparands, comparand => comparand.Statistics?.Mean ?? double.NaN).ToArray(); + + foreach (var kvp in CustomStatisticsCalculationMap) + { + string customStatistics = kvp.Key; + Func calculation = kvp.Value; + OriginalBaselineCustomStatistics[customStatistics] = Array.Empty(); + OriginalComparandCustomStatistics[customStatistics] = Array.Empty(); + + OriginalBaselineCustomStatistics[customStatistics] = + GoodLinq.Select(baselines, baseline => calculation(baseline.Statistics) ?? double.NaN).ToArray(); + OriginalComparandCustomStatistics[customStatistics] = + GoodLinq.Select(comparands, comparand => calculation(comparand.Statistics) ?? double.NaN).ToArray(); + } } public List ComparisonResults { get; set; } @@ -76,44 +132,88 @@ public double MeanDiffPerc{ } } - //public double? GetDiffPercentFromOtherMetrics(string metricName) - //{ - // List baselineOtherMetricCollection = new(); - // List comparandOtherMetricCollection = new(); - - // foreach (var baseline in Baselines) - // { - // if (baseline.OtherMetrics.TryGetValue(metricName, out var baselineMetric)) - // { - // if (baselineMetric.HasValue) - // { - // baselineOtherMetricCollection.Add(baselineMetric.Value); - // } - // } - // } - - // foreach (var comparand in Comparands) - // { - // if (comparand.OtherMetrics.TryGetValue(metricName, out var comparandMetric)) - // { - // if (comparandMetric.HasValue) - // { - // comparandOtherMetricCollection.Add(comparandMetric.Value); - // } - // } - // } - - // if (baselineOtherMetricCollection.Count() * comparandOtherMetricCollection.Count() == 0) - // { - // return null; - // } - - // var outliersFreeBaselineOtherMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(baselineOtherMetricCollection); - // var outliersFreeComparandOtherMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(comparandOtherMetricCollection); - - // var averagedOutliersFreeBaselineOtherMetric = outliersFreeBaselineOtherMetricCollection.Average(); - // var averagedOutliersFreeComparandOtherMetric = outliersFreeComparandOtherMetricCollection.Average(); - // return (averagedOutliersFreeBaselineOtherMetric - averagedOutliersFreeComparandOtherMetric) / averagedOutliersFreeBaselineOtherMetric; - //} + public Dictionary OriginalBaselineCustomStatistics { get; } = new(); + public Dictionary OriginalComparandCustomStatistics { get; } = new(); + public Dictionary OutliersFreeBaselineCustomStatistics => + GoodLinq.Select(OriginalBaselineCustomStatistics, kvp => + (kvp.Key, GC.Analysis.API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + .ToDictionary(); + public Dictionary OutliersFreeComparandCustomStatistics => + GoodLinq.Select(OriginalComparandCustomStatistics, kvp => + (kvp.Key, GC.Analysis.API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + .ToDictionary(); + public Dictionary AveragedBaselineCustomStatistics => + GoodLinq.Select(OutliersFreeBaselineCustomStatistics, kvp => + (kvp.Key, GoodLinq.Average(kvp.Value, v => v))) + .ToDictionary(); + public Dictionary AveragedComparandCustomStatistics => + GoodLinq.Select(OutliersFreeComparandCustomStatistics, kvp => + (kvp.Key, GoodLinq.Average(kvp.Value, v => v))) + .ToDictionary(); + + public Dictionary CustomStatisticsDiff => + GoodLinq.Select(OutliersFreeBaselineCustomStatistics, kvp => + (kvp.Key, AveragedComparandCustomStatistics[kvp.Key] - AveragedBaselineCustomStatistics[kvp.Key])) + .ToDictionary(); + + public Dictionary CustomStatisticsDiffPerc => + GoodLinq.Select(OutliersFreeBaselineCustomStatistics, kvp => + { + if (AveragedBaselineCustomStatistics[kvp.Key] == 0) + { + if (AveragedComparandCustomStatistics[kvp.Key] == 0) + { + return (kvp.Key, 0); + } + else + { + return (kvp.Key, double.NaN); + } + } + return (kvp.Key, CustomStatisticsDiff[kvp.Key] / AveragedBaselineCustomStatistics[kvp.Key]); + }) + .ToDictionary(); + + public Dictionary OriginalBaselineCustomGCData { get; } = new(); + public Dictionary OriginalComparandCustomGCData { get; } = new(); + public Dictionary OutliersFreeBaselineCustomGCData => + GoodLinq.Select(OriginalBaselineCustomGCData, kvp => + (kvp.Key, GC.Analysis.API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + .ToDictionary(); + public Dictionary OutliersFreeComparandCustomGCData => + GoodLinq.Select(OriginalComparandCustomGCData, kvp => + (kvp.Key, GC.Analysis.API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + .ToDictionary(); + public Dictionary AveragedBaselineCustomGCData => + GoodLinq.Select(OutliersFreeBaselineCustomGCData, kvp => + (kvp.Key, GoodLinq.Average(kvp.Value, v => v))) + .ToDictionary(); + public Dictionary AveragedComparandCustomGCData => + GoodLinq.Select(OutliersFreeComparandCustomGCData, kvp => + (kvp.Key, GoodLinq.Average(kvp.Value, v => v))) + .ToDictionary(); + + public Dictionary CustomGCDataDiff => + GoodLinq.Select(OutliersFreeBaselineCustomGCData, kvp => + (kvp.Key, AveragedComparandCustomGCData[kvp.Key] - AveragedBaselineCustomGCData[kvp.Key])) + .ToDictionary(); + + public Dictionary CustomGCDataDiffPerc => + GoodLinq.Select(OutliersFreeBaselineCustomGCData, kvp => + { + if (AveragedBaselineCustomGCData[kvp.Key] == 0) + { + if (AveragedComparandCustomGCData[kvp.Key] == 0) + { + return (kvp.Key, 0); + } + else + { + return (kvp.Key, double.NaN); + } + } + return (kvp.Key, CustomGCDataDiff[kvp.Key] / AveragedBaselineCustomGCData[kvp.Key]); + }) + .ToDictionary(); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs index 128d05be573..2763dbfb6a1 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -1,57 +1,16 @@ using GC.Analysis.API; using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using Newtonsoft.Json; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { public sealed class MicrobenchmarkResult { public Statistics? Statistics { get; set; } - - [JsonIgnore] public GCProcessData? GCData { get; set; } - public GCTraceMetrics? GCTraceMetrics { get; set; } - - [JsonIgnore] public CPUProcessData? CPUData { get; set; } public Run? Parent { get; set; } public string? MicrobenchmarkName { get; set; } - public Dictionary OtherMetrics { get; set; } = new(); - - private static readonly IReadOnlyDictionary> _customStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - { "number of iterations", (Statistics stats) => stats.N }, - { "min", (Statistics stats) => stats.Min }, - { "max", (Statistics stats) => stats.Max }, - { "median", (Statistics stats) => stats.Median }, - { "q1", (Statistics stats) => stats.Q1 }, - { "q3", (Statistics stats) => stats.Q3 }, - { "variance", (Statistics stats) => stats.Variance }, - { "standard deviation", (Statistics stats) => stats.StandardDeviation }, - { "skewness", (Statistics stats) => stats.Skewness }, - { "kurtosis", (Statistics stats) => stats.Kurtosis }, - { "standard error", (Statistics stats) => stats.StandardError }, - { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, - }; - - public static double? LookupStatisticsCalculation(string columnName, MicrobenchmarkResult result) - { - if (string.IsNullOrEmpty(columnName)) - { - return null; - } - - if (!_customStatisticsCalculationMap.TryGetValue(columnName, out var val)) - { - return null; - } - - else - { - return val.Invoke(result.Statistics); - } - } } } \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index d1237a99d86..03eace3be9c 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -8,27 +8,26 @@ namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Markdown { - /* private const string baseTableString = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; private const string baseTableRows = "| --- | --- | -- | --- | --- | --- | --- | "; - public static void GenerateTable(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResults, Dictionary executionDetails, string path) + public static void GenerateTable(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResultsCollection, Dictionary executionDetails, string path) { using (StreamWriter sw = new StreamWriter(path)) { // Create summary. sw.WriteLine("# Summary"); - string header = $"| Criteria | {string.Join("|", GoodLinq.Select(comparisonResults, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; + string header = $"| Criteria | {string.Join("|", GoodLinq.Select(comparisonResultsCollection, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; sw.WriteLine(header); - sw.WriteLine($"| ----- | {string.Join("|", Enumerable.Repeat(" ----- ", comparisonResults.Count))} |"); - sw.WriteLine($"| Large Regressions (>20%) | {GoodLinq.Sum(comparisonResults, s => s.LargeRegressions.Count())}|"); - sw.WriteLine($"| Regressions (5% - 20%) | {GoodLinq.Sum(comparisonResults, s => s.Regressions.Count())}|"); - sw.WriteLine($"| Stale Regressions (0% - 5%) | {GoodLinq.Sum(comparisonResults, s => s.StaleRegressions.Count())}|"); - sw.WriteLine($"| Stale Improvements (0% - 5%) | {GoodLinq.Sum(comparisonResults, s => s.StaleImprovements.Count())}|"); - sw.WriteLine($"| Improvements (5% - 20%) | {GoodLinq.Sum(comparisonResults, s => s.Improvements.Count())}|"); - sw.WriteLine($"| Large Improvements (>20%) | {GoodLinq.Sum(comparisonResults, s => s.LargeImprovements.Count())}|"); - sw.WriteLine($"| Total | {comparisonResults.Count} |"); + sw.WriteLine($"| ----- | {string.Join("|", Enumerable.Repeat(" ----- ", comparisonResultsCollection.Count))} |"); + sw.WriteLine($"| Large Regressions (>20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.LargeRegressions.Count())}|"); + sw.WriteLine($"| Regressions (5% - 20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.Regressions.Count())}|"); + sw.WriteLine($"| Stale Regressions (0% - 5%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.StaleRegressions.Count())}|"); + sw.WriteLine($"| Stale Improvements (0% - 5%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.StaleImprovements.Count())}|"); + sw.WriteLine($"| Improvements (5% - 20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.Improvements.Count())}|"); + sw.WriteLine($"| Large Improvements (>20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.LargeImprovements.Count())}|"); + sw.WriteLine($"| Total | {comparisonResultsCollection.Count} |"); sw.WriteLine("\n"); // Incomplete Tests. @@ -42,9 +41,9 @@ public static void GenerateTable(MicrobenchmarkConfiguration configuration, IRea sw.WriteLine("## Individual Results"); // Add details of Each Comparison. - foreach (var comparisonResult in comparisonResults) + foreach (var comparisonResults in comparisonResultsCollection) { - AddDetailsOfSingleComparison(sw, configuration, comparisonResult); + AddDetailsOfSingleComparison(sw, configuration, comparisonResults); sw.WriteLine("\n"); } } @@ -85,50 +84,50 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben sw.AddTableForSingleCriteria(configuration, comparisonResult.StaleImprovements); sw.WriteLine("\n\n"); - if (configuration.Output.additional_report_metrics != null) - { - foreach (var metric in configuration.Output.additional_report_metrics) - { - sw.WriteLine($"## Comparison by {metric}"); - var ordered = comparisonResult.Comparisons.OrderByDescending(c => c.GetDiffPercentFromOtherMetrics(metric)); - - // Large Regressions - sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.2)); - sw.WriteLine("\n"); - - // Large Improvements - sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); - var largeImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < -0.2); - largeImprovements.Reverse(); - sw.AddTableForSingleCriteria(configuration, largeImprovements); - sw.WriteLine("\n"); - - // Regressions - sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2)); - sw.WriteLine("\n"); - - // Improvements - sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); - var improvements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2); - improvements.Reverse(); - sw.AddTableForSingleCriteria(configuration, improvements); - sw.WriteLine("\n"); - - // Stale Regressions - sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < 0.05 && o.GetDiffPercentFromOtherMetrics(metric) >= 0.0)); - sw.WriteLine("\n"); - - // Stale Improvements - sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); - var staleImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > -0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.0); - staleImprovements.Reverse(); - sw.AddTableForSingleCriteria(configuration, staleImprovements); - sw.WriteLine("\n"); - } - } + //if (configuration.Output.additional_report_metrics != null) + //{ + // foreach (var metric in configuration.Output.additional_report_metrics) + // { + // sw.WriteLine($"## Comparison by {metric}"); + // var ordered = comparisonResult.Comparisons.OrderByDescending(c => c.GetDiffPercentFromOtherMetrics(metric)); + + // // Large Regressions + // sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); + // sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.2)); + // sw.WriteLine("\n"); + + // // Large Improvements + // sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); + // var largeImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < -0.2); + // largeImprovements.Reverse(); + // sw.AddTableForSingleCriteria(configuration, largeImprovements); + // sw.WriteLine("\n"); + + // // Regressions + // sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); + // sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2)); + // sw.WriteLine("\n"); + + // // Improvements + // sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); + // var improvements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2); + // improvements.Reverse(); + // sw.AddTableForSingleCriteria(configuration, improvements); + // sw.WriteLine("\n"); + + // // Stale Regressions + // sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); + // sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < 0.05 && o.GetDiffPercentFromOtherMetrics(metric) >= 0.0)); + // sw.WriteLine("\n"); + + // // Stale Improvements + // sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); + // var staleImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > -0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.0); + // staleImprovements.Reverse(); + // sw.AddTableForSingleCriteria(configuration, staleImprovements); + // sw.WriteLine("\n"); + // } + //} } internal static void AddTableForSingleCriteria(this StreamWriter sw, MicrobenchmarkConfiguration configuration, IEnumerable comparisons) @@ -162,24 +161,28 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm { try { - var baseRow = $"| {lr.MicrobenchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.Baseline.Statistics.Mean.Value, 2)} | {Math.Round(lr.Comparand.Statistics.Mean.Value, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; + string benchmarkName = lr.MicrobenchmarkName.Replace("<", "\\<").Replace(">", "\\>"); + var baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineMeanValue, 2)} | {Math.Round(lr.AveragedComparandMeanValue, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; if (configuration.Output.Columns != null) { foreach (var column in configuration.Output.Columns) { - if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) + double? baselineValue = null; + double? comparandValue = null; + if (MicrobenchmarkComparisonResult.CustomStatisticsCalculationMap.Keys.Contains(column)) { - lr.Baseline.OtherMetrics[column] = baselineValue = API.GCProcessData.LookupAggregateCalculation(column, lr.Baseline.GCData) ?? MicrobenchmarkResult.LookupStatisticsCalculation(column, lr.Baseline); + baselineValue = lr.AveragedBaselineCustomStatistics.GetValueOrDefault(column); + comparandValue = lr.AveragedComparandCustomStatistics.GetValueOrDefault(column); } - string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 4).ToString() : string.Empty; - - if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) + if (MicrobenchmarkComparisonResult.CustomAggregateCalculationMap.Keys.Contains(column)) { - lr.Comparand.OtherMetrics[column] = comparandValue = API.GCProcessData.LookupAggregateCalculation(column, lr.Comparand.GCData) ?? MicrobenchmarkResult.LookupStatisticsCalculation(column, lr.Comparand); + baselineValue = lr.AveragedBaselineCustomGCData.GetValueOrDefault(column); + comparandValue = lr.AveragedComparandCustomGCData.GetValueOrDefault(column); } + + string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 4).ToString() : string.Empty; string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 4).ToString() : string.Empty; - double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; string deltaResult = delta.HasValue ? Math.Round(delta.Value, 4).ToString() : string.Empty; @@ -190,31 +193,31 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm } } - if (configuration.Output.cpu_columns != null) - { - foreach (var column in configuration.Output.cpu_columns) - { - if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) - { - lr.Baseline.OtherMetrics[column] = baselineValue = lr.Baseline.CPUData?.GetIncCountForGCMethod(column) ?? null; - } - string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 2).ToString() : string.Empty; + //if (configuration.Output.cpu_columns != null) + //{ + // foreach (var column in configuration.Output.cpu_columns) + // { + // if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) + // { + // lr.Baseline.OtherMetrics[column] = baselineValue = lr.Baseline.CPUData?.GetIncCountForGCMethod(column) ?? null; + // } + // string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 2).ToString() : string.Empty; - if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) - { - lr.Comparand.OtherMetrics[column] = comparandValue = lr.Comparand.CPUData?.GetIncCountForGCMethod(column) ?? null; - } - string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 2).ToString() : string.Empty; + // if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) + // { + // lr.Comparand.OtherMetrics[column] = comparandValue = lr.Comparand.CPUData?.GetIncCountForGCMethod(column) ?? null; + // } + // string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 2).ToString() : string.Empty; - double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; - string deltaResult = delta.HasValue ? Math.Round(delta.Value, 2).ToString() : string.Empty; + // double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; + // string deltaResult = delta.HasValue ? Math.Round(delta.Value, 2).ToString() : string.Empty; - double? deltaPercent = delta.HasValue ? (delta / baselineValue.Value) * 100 : null; - string deltaPercentResult = deltaPercent.HasValue ? Math.Round(deltaPercent.Value, 2).ToString() : string.Empty; + // double? deltaPercent = delta.HasValue ? (delta / baselineValue.Value) * 100 : null; + // string deltaPercentResult = deltaPercent.HasValue ? Math.Round(deltaPercent.Value, 2).ToString() : string.Empty; - baseRow += $"{baselineResult} | {comparandResult} | {deltaResult} | {deltaPercentResult} |"; - } - } + // baseRow += $"{baselineResult} | {comparandResult} | {deltaResult} | {deltaPercentResult} |"; + // } + //} sw.WriteLine(baseRow); } @@ -225,16 +228,6 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm Console.WriteLine(e.StackTrace); } } - - // Dispose all the Analyzers now that we are persisting the values. - foreach (var comparison in comparisons) - { - comparison.Baseline?.GCData?.Parent?.Dispose(); - comparison.Baseline?.CPUData?.Parent?.Analyzer?.Dispose(); - comparison.Comparand?.GCData?.Parent?.Dispose(); - comparison.Comparand?.CPUData?.Parent?.Analyzer?.Dispose(); - } } - */ } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs index cb0fd4db214..8d879a48ecd 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs @@ -12,7 +12,7 @@ public static void Present(MicrobenchmarkConfiguration configuration, List Date: Sat, 9 May 2026 14:20:35 +0800 Subject: [PATCH 17/54] get value from StatsData if property is not found in GCTraceMetrics --- .../Analysis/GCTraceMetricComparisonResult.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs index 0f6d69eb604..568252b6317 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs @@ -1,6 +1,7 @@ using GC.Analysis.API; using Microsoft.Diagnostics.Tracing.Analysis.GC; using System.Reflection; +using System.Runtime.Serialization.Formatters; namespace GC.Infrastructure.Core.Analysis { @@ -43,15 +44,15 @@ public GCTraceMetricComparisonResult(IEnumerable baselines, IEnu else { - OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => (double)fieldInfo.GetValue(baseline)); - OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => (double)fieldInfo.GetValue(comparand)); + OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => baseline.StatsData[fieldInfo.Name]); + OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => comparand.StatsData[fieldInfo.Name]); } } else { - OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => (double)pInfo.GetValue(baseline)); - OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => (double)pInfo.GetValue(comparand)); + OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => baseline.StatsData[pInfo.Name]); + OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => comparand.StatsData[pInfo.Name]); } } From 69ea04d4a1abb7a5aa3fccc387e249719f8b9576 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Sat, 9 May 2026 15:18:30 +0800 Subject: [PATCH 18/54] filtering to finite values --- .../gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs index a03cf8fb15b..d0269d32326 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs @@ -38,9 +38,12 @@ public static IEnumerable RemoveOutliers(IEnumerable collection) { return Array.Empty(); } + double[] validCollection = collection + .Where(x => !double.IsNaN(x) && !double.IsInfinity(x)) + .ToArray(); // Calculate Q1 (25th percentile) and Q3 (75th percentile) - double q1 = GC.Analysis.API.Statistics.Percentile(collection, 0.25); - double q3 = GC.Analysis.API.Statistics.Percentile(collection, 0.75); + double q1 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.25); + double q3 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.75); // Calculate IQR (Interquartile Range) double iqr = q3 - q1; From 30c9c9170114ed604e68cf76ee2d5c1ca67f8e26 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Sat, 9 May 2026 15:38:53 +0800 Subject: [PATCH 19/54] check possible null references --- .../Analysis/GCTraceMetricComparisonResult.cs | 19 ++- .../MicrobenchmarkComparisonResult.cs | 123 ++++++++++-------- .../MicrobenchmarkResultComparison.cs | 3 +- 3 files changed, 78 insertions(+), 67 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs index 568252b6317..71a07603099 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs @@ -1,7 +1,6 @@ -using GC.Analysis.API; +using API = GC.Analysis.API; using Microsoft.Diagnostics.Tracing.Analysis.GC; using System.Reflection; -using System.Runtime.Serialization.Formatters; namespace GC.Infrastructure.Core.Analysis { @@ -18,8 +17,8 @@ public GCTraceMetricComparisonResult(IEnumerable baselines, IEnu // Property found on the GCTraceMetrics. if (pInfo != null) { - OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => (double)pInfo.GetValue(baseline)); - OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => (double)pInfo.GetValue(comparand)); + OriginalBaselineMetricCollection = baselines.Select(baseline => (double)pInfo.GetValue(baseline)); + OriginalComparandMetricCollection = comparands.Select(comparand => (double)pInfo.GetValue(comparand)); } // If property isn't found on the GCTraceMetrics, look in GCStats. @@ -44,15 +43,15 @@ public GCTraceMetricComparisonResult(IEnumerable baselines, IEnu else { - OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => baseline.StatsData[fieldInfo.Name]); - OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => comparand.StatsData[fieldInfo.Name]); + OriginalBaselineMetricCollection = baselines.Select(baseline => baseline.StatsData[fieldInfo.Name]); + OriginalComparandMetricCollection = comparands.Select(comparand => comparand.StatsData[fieldInfo.Name]); } } else { - OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => baseline.StatsData[pInfo.Name]); - OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => comparand.StatsData[pInfo.Name]); + OriginalBaselineMetricCollection = baselines.Select(baseline => baseline.StatsData[pInfo.Name]); + OriginalComparandMetricCollection = comparands.Select(comparand => comparand.StatsData[pInfo.Name]); } } @@ -61,8 +60,8 @@ public GCTraceMetricComparisonResult(IEnumerable baselines, IEnu OutliersFreeComparandMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(OriginalComparandMetricCollection); // Calculate averaged metrics - AveragedBaselineMetric = GoodLinq.Average(OutliersFreeBaselineMetricCollection, r => r); - AveragedComparandMetric = GoodLinq.Average(OutliersFreeComparandMetricCollection, r => r); + AveragedBaselineMetric = API.GoodLinq.Average(OutliersFreeBaselineMetricCollection, r => r); + AveragedComparandMetric = API.GoodLinq.Average(OutliersFreeComparandMetricCollection, r => r); } public string RunName { get; } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 6ee5218dad5..ee9c1477965 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -1,4 +1,4 @@ -using GC.Analysis.API; +using API = GC.Analysis.API; using Microsoft.Diagnostics.Tracing.Parsers.Clr; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks @@ -31,7 +31,7 @@ public sealed class MicrobenchmarkComparisonResult { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, }; - public static readonly Dictionary> CustomAggregateCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + public static readonly Dictionary> CustomAggregateCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) { { "gc count", (gc) => gc.Stats.Count }, { "non induced gc count", (gc) => gc.Stats.Count - gc.GCs.Count(g => g.Reason == GCReason.Induced)}, @@ -40,8 +40,8 @@ public sealed class MicrobenchmarkComparisonResult { "max size peak (mb)", (gc) => gc.Stats.MaxSizePeakMB }, { "total pause time (msec)", (gc) => gc.Stats.TotalPauseTimeMSec }, { "gc pause time %", (gc) => gc.Stats.GetGCPauseTimePercentage() }, - { "avg. heap size (mb)", (gc) => GoodLinq.Average(gc.GCs, g => g.HeapSizeBeforeMB) }, - { "avg. heap size after (mb)", (gc) => GoodLinq.Average(gc.GCs, g => g.HeapSizeAfterMB) }, + { "avg. heap size (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeBeforeMB) }, + { "avg. heap size after (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeAfterMB) }, }; public MicrobenchmarkComparisonResult() { } @@ -51,26 +51,40 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline ComparisonResults = new(); if (includeTraces) { - var baselineGCTraceMetricsCollection = GoodLinq.Select(baselines, baseline => baseline.GCTraceMetrics); - var comparandGCTraceMetricsCollection = GoodLinq.Select(comparands, comparand => comparand.GCTraceMetrics); + var baselineGCTraceMetricsCollection = baselines + .Where(baseline => baseline != null) + .Select(baseline => baseline.GCTraceMetrics) + .ToArray(); - foreach (var metricName in RequiredMetrics) + var comparandGCTraceMetricsCollection = comparands + .Where(comparand => comparand != null) + .Select(comparand => comparand.GCTraceMetrics) + .ToArray(); + + if (baselineGCTraceMetricsCollection.Length > 0 && comparandGCTraceMetricsCollection.Length > 0) { - ComparisonResults.Add( - GCTraceMetricComparison.CompareGCTraceMetric(baselineGCTraceMetricsCollection, comparandGCTraceMetricsCollection, metricName)); + foreach (var metricName in RequiredMetrics) + { + ComparisonResults.Add( + GCTraceMetricComparison.CompareGCTraceMetric( + baselineGCTraceMetricsCollection!, comparandGCTraceMetricsCollection!, metricName)); + } + } foreach (var kvp in CustomAggregateCalculationMap) { string customGCData = kvp.Key; - Func calculation = kvp.Value; + Func calculation = kvp.Value; OriginalBaselineCustomGCData[customGCData] = Array.Empty(); OriginalComparandCustomGCData[customGCData] = Array.Empty(); - OriginalBaselineCustomGCData[customGCData] = - GoodLinq.Select(baselines, baseline => calculation(baseline.GCData)).ToArray(); - OriginalComparandCustomGCData[customGCData] = - GoodLinq.Select(comparands, comparand => calculation(comparand.GCData)).ToArray(); + OriginalBaselineCustomGCData[customGCData] = baselines + .Select(baseline => calculation(baseline.GCData)) + .ToArray(); + OriginalComparandCustomGCData[customGCData] = comparands + .Select(comparand => calculation(comparand.GCData)) + .ToArray(); } } @@ -78,10 +92,12 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline ComparandRunName = comparands?.FirstOrDefault()?.Parent?.Name; MicrobenchmarkName = baselines?.FirstOrDefault()?.MicrobenchmarkName; - OriginalBaselineMeanValueCollection = - GoodLinq.Select(baselines, baseline => baseline.Statistics?.Mean ?? double.NaN).ToArray(); - OriginalComparandMeanValueCollection = - GoodLinq.Select(comparands, comparand => comparand.Statistics?.Mean ?? double.NaN).ToArray(); + OriginalBaselineMeanValueCollection = baselines + .Select(baseline => baseline.Statistics?.Mean ?? double.NaN) + .ToArray(); + OriginalComparandMeanValueCollection = comparands + .Select(comparand => comparand.Statistics?.Mean ?? double.NaN) + .ToArray(); foreach (var kvp in CustomStatisticsCalculationMap) { @@ -90,10 +106,12 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline OriginalBaselineCustomStatistics[customStatistics] = Array.Empty(); OriginalComparandCustomStatistics[customStatistics] = Array.Empty(); - OriginalBaselineCustomStatistics[customStatistics] = - GoodLinq.Select(baselines, baseline => calculation(baseline.Statistics) ?? double.NaN).ToArray(); - OriginalComparandCustomStatistics[customStatistics] = - GoodLinq.Select(comparands, comparand => calculation(comparand.Statistics) ?? double.NaN).ToArray(); + OriginalBaselineCustomStatistics[customStatistics] = baselines + .Select(baseline => calculation(baseline.Statistics) ?? double.NaN) + .ToArray(); + OriginalComparandCustomStatistics[customStatistics] = comparands + .Select(comparand => calculation(comparand.Statistics) ?? double.NaN) + .ToArray(); } } @@ -110,8 +128,8 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline public double[] OutliersFreeComparandMeanValueCollection => GC.Analysis.API.Statistics.RemoveOutliers(OriginalComparandMeanValueCollection).ToArray(); - public double AveragedBaselineMeanValue => GoodLinq.Average(OutliersFreeBaselineMeanValueCollection, r => r); - public double AveragedComparandMeanValue => GoodLinq.Average(OutliersFreeComparandMeanValueCollection, r => r); + public double AveragedBaselineMeanValue => API.GoodLinq.Average(OutliersFreeBaselineMeanValueCollection, r => r); + public double AveragedComparandMeanValue => API.GoodLinq.Average(OutliersFreeComparandMeanValueCollection, r => r); public double MeanDiff => AveragedComparandMeanValue - AveragedBaselineMeanValue; public double MeanDiffPerc{ @@ -134,30 +152,25 @@ public double MeanDiffPerc{ public Dictionary OriginalBaselineCustomStatistics { get; } = new(); public Dictionary OriginalComparandCustomStatistics { get; } = new(); - public Dictionary OutliersFreeBaselineCustomStatistics => - GoodLinq.Select(OriginalBaselineCustomStatistics, kvp => - (kvp.Key, GC.Analysis.API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + public Dictionary OutliersFreeBaselineCustomStatistics => OriginalBaselineCustomStatistics + .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) .ToDictionary(); - public Dictionary OutliersFreeComparandCustomStatistics => - GoodLinq.Select(OriginalComparandCustomStatistics, kvp => - (kvp.Key, GC.Analysis.API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + public Dictionary OutliersFreeComparandCustomStatistics => OriginalComparandCustomStatistics + .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) .ToDictionary(); - public Dictionary AveragedBaselineCustomStatistics => - GoodLinq.Select(OutliersFreeBaselineCustomStatistics, kvp => - (kvp.Key, GoodLinq.Average(kvp.Value, v => v))) + public Dictionary AveragedBaselineCustomStatistics => OutliersFreeBaselineCustomStatistics + .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) .ToDictionary(); - public Dictionary AveragedComparandCustomStatistics => - GoodLinq.Select(OutliersFreeComparandCustomStatistics, kvp => - (kvp.Key, GoodLinq.Average(kvp.Value, v => v))) + public Dictionary AveragedComparandCustomStatistics => OutliersFreeComparandCustomStatistics + .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) .ToDictionary(); - public Dictionary CustomStatisticsDiff => - GoodLinq.Select(OutliersFreeBaselineCustomStatistics, kvp => - (kvp.Key, AveragedComparandCustomStatistics[kvp.Key] - AveragedBaselineCustomStatistics[kvp.Key])) + public Dictionary CustomStatisticsDiff => OutliersFreeBaselineCustomStatistics + .Select(kvp => (kvp.Key, AveragedComparandCustomStatistics[kvp.Key] - AveragedBaselineCustomStatistics[kvp.Key])) .ToDictionary(); - public Dictionary CustomStatisticsDiffPerc => - GoodLinq.Select(OutliersFreeBaselineCustomStatistics, kvp => + public Dictionary CustomStatisticsDiffPerc => OutliersFreeBaselineCustomStatistics + .Select(kvp => { if (AveragedBaselineCustomStatistics[kvp.Key] == 0) { @@ -176,30 +189,30 @@ public double MeanDiffPerc{ public Dictionary OriginalBaselineCustomGCData { get; } = new(); public Dictionary OriginalComparandCustomGCData { get; } = new(); - public Dictionary OutliersFreeBaselineCustomGCData => - GoodLinq.Select(OriginalBaselineCustomGCData, kvp => + public Dictionary OutliersFreeBaselineCustomGCData => OriginalBaselineCustomGCData + .Select(kvp => (kvp.Key, GC.Analysis.API.Statistics.RemoveOutliers(kvp.Value).ToArray())) .ToDictionary(); - public Dictionary OutliersFreeComparandCustomGCData => - GoodLinq.Select(OriginalComparandCustomGCData, kvp => + public Dictionary OutliersFreeComparandCustomGCData => OriginalComparandCustomGCData + .Select(kvp => (kvp.Key, GC.Analysis.API.Statistics.RemoveOutliers(kvp.Value).ToArray())) .ToDictionary(); - public Dictionary AveragedBaselineCustomGCData => - GoodLinq.Select(OutliersFreeBaselineCustomGCData, kvp => - (kvp.Key, GoodLinq.Average(kvp.Value, v => v))) + public Dictionary AveragedBaselineCustomGCData => OutliersFreeBaselineCustomGCData + .Select(kvp => + (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) .ToDictionary(); - public Dictionary AveragedComparandCustomGCData => - GoodLinq.Select(OutliersFreeComparandCustomGCData, kvp => - (kvp.Key, GoodLinq.Average(kvp.Value, v => v))) + public Dictionary AveragedComparandCustomGCData => OutliersFreeComparandCustomGCData + .Select(kvp => + (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) .ToDictionary(); - public Dictionary CustomGCDataDiff => - GoodLinq.Select(OutliersFreeBaselineCustomGCData, kvp => + public Dictionary CustomGCDataDiff => OutliersFreeBaselineCustomGCData + .Select(kvp => (kvp.Key, AveragedComparandCustomGCData[kvp.Key] - AveragedBaselineCustomGCData[kvp.Key])) .ToDictionary(); - public Dictionary CustomGCDataDiffPerc => - GoodLinq.Select(OutliersFreeBaselineCustomGCData, kvp => + public Dictionary CustomGCDataDiffPerc => OutliersFreeBaselineCustomGCData + .Select(kvp => { if (AveragedBaselineCustomGCData[kvp.Key] == 0) { diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 182b838a67f..0a27bc2786b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -66,8 +66,7 @@ public static ConcurrentDictionary> MapBenchmarkFu string? fullName = results?.Benchmarks?.FirstOrDefault()?.FullName; if (fullName != null) { - benchmarkFullNameJsonMap[fullName] = benchmarkFullNameJsonMap.GetValueOrDefault(fullName, new()); - benchmarkFullNameJsonMap[fullName].Add(jsonFile); + benchmarkFullNameJsonMap.GetOrAdd(fullName, _ => new ConcurrentBag()).Add(jsonFile); } }); From 3b38059911d9360c491b2b7d9f9949052ffe1b99 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Sat, 9 May 2026 15:57:28 +0800 Subject: [PATCH 20/54] add console output when analyzing results --- .../Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index b1b4c184b88..af0af6f7da2 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -1,7 +1,9 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.Presentation.Microbenchmarks; +using Spectre.Console; using Spectre.Console.Cli; using System.ComponentModel; using System.Diagnostics.CodeAnalysis; @@ -41,6 +43,7 @@ public static List ExecuteAnalysis(Microbenchma foreach (var benchmarkFullName in benchmarkFullNameJsonMap.Keys) { + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Analyzing Microbenchmarks: {benchmarkFullName} [/]\n"); List comparisonResultsForBenchmark = MicrobenchmarkResultComparison.CompareMicrobenchmarkResultForBenchmark(configuration, benchmarkFullName); comparisonResultForAllBenchmarks.AddRange(comparisonResultsForBenchmark); } From f9c6d08605dbcff2d15077385a9b7ba3fd3f25cd Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Tue, 12 May 2026 15:26:56 +0800 Subject: [PATCH 21/54] redesign microbenchmarks result --- .../Analysis/GCTraceMetricComparisonResult.cs | 4 +- .../MicrobenchmarkComparisonResult.cs | 145 +++--------------- .../Microbenchmarks/MicrobenchmarkResult.cs | 88 ++++++++++- .../MicrobenchmarkResultComparison.cs | 98 ++++++------ .../Presentation/Microbenchmarks/Markdown.cs | 98 +++++------- .../MicrobenchmarkAnalyzeCommand.cs | 21 ++- 6 files changed, 210 insertions(+), 244 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs index 71a07603099..deb14e44080 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs @@ -56,8 +56,8 @@ public GCTraceMetricComparisonResult(IEnumerable baselines, IEnu } // Filter out outliers using IQR method - OutliersFreeBaselineMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(OriginalBaselineMetricCollection); - OutliersFreeComparandMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(OriginalComparandMetricCollection); + OutliersFreeBaselineMetricCollection = API.Statistics.RemoveOutliers(OriginalBaselineMetricCollection); + OutliersFreeComparandMetricCollection = API.Statistics.RemoveOutliers(OriginalComparandMetricCollection); // Calculate averaged metrics AveragedBaselineMetric = API.GoodLinq.Average(OutliersFreeBaselineMetricCollection, r => r); diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index ee9c1477965..ade87c6d887 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -15,35 +15,6 @@ public sealed class MicrobenchmarkComparisonResult "PauseDurationMSec_MeanWhereIsBlockingGen2" }; - public static readonly IReadOnlyDictionary> CustomStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - { "number of iterations", (Statistics stats) => stats.N }, - { "min", (Statistics stats) => stats.Min }, - { "max", (Statistics stats) => stats.Max }, - { "median", (Statistics stats) => stats.Median }, - { "q1", (Statistics stats) => stats.Q1 }, - { "q3", (Statistics stats) => stats.Q3 }, - { "variance", (Statistics stats) => stats.Variance }, - { "standard deviation", (Statistics stats) => stats.StandardDeviation }, - { "skewness", (Statistics stats) => stats.Skewness }, - { "kurtosis", (Statistics stats) => stats.Kurtosis }, - { "standard error", (Statistics stats) => stats.StandardError }, - { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, - }; - - public static readonly Dictionary> CustomAggregateCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - { "gc count", (gc) => gc.Stats.Count }, - { "non induced gc count", (gc) => gc.Stats.Count - gc.GCs.Count(g => g.Reason == GCReason.Induced)}, - { "induced gc count", (gc) => gc.GCs.Count(g => g.Reason == GCReason.Induced)}, - { "total allocated (mb)", (gc) => gc.Stats.TotalAllocatedMB }, - { "max size peak (mb)", (gc) => gc.Stats.MaxSizePeakMB }, - { "total pause time (msec)", (gc) => gc.Stats.TotalPauseTimeMSec }, - { "gc pause time %", (gc) => gc.Stats.GetGCPauseTimePercentage() }, - { "avg. heap size (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeBeforeMB) }, - { "avg. heap size after (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeAfterMB) }, - }; - public MicrobenchmarkComparisonResult() { } public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands, bool includeTraces = true) @@ -71,48 +42,14 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline } } - - foreach (var kvp in CustomAggregateCalculationMap) - { - string customGCData = kvp.Key; - Func calculation = kvp.Value; - OriginalBaselineCustomGCData[customGCData] = Array.Empty(); - OriginalComparandCustomGCData[customGCData] = Array.Empty(); - - OriginalBaselineCustomGCData[customGCData] = baselines - .Select(baseline => calculation(baseline.GCData)) - .ToArray(); - OriginalComparandCustomGCData[customGCData] = comparands - .Select(comparand => calculation(comparand.GCData)) - .ToArray(); - } } BaselineRunName = baselines?.FirstOrDefault()?.Parent?.Name; ComparandRunName = comparands?.FirstOrDefault()?.Parent?.Name; MicrobenchmarkName = baselines?.FirstOrDefault()?.MicrobenchmarkName; - OriginalBaselineMeanValueCollection = baselines - .Select(baseline => baseline.Statistics?.Mean ?? double.NaN) - .ToArray(); - OriginalComparandMeanValueCollection = comparands - .Select(comparand => comparand.Statistics?.Mean ?? double.NaN) - .ToArray(); - - foreach (var kvp in CustomStatisticsCalculationMap) - { - string customStatistics = kvp.Key; - Func calculation = kvp.Value; - OriginalBaselineCustomStatistics[customStatistics] = Array.Empty(); - OriginalComparandCustomStatistics[customStatistics] = Array.Empty(); - - OriginalBaselineCustomStatistics[customStatistics] = baselines - .Select(baseline => calculation(baseline.Statistics) ?? double.NaN) - .ToArray(); - OriginalComparandCustomStatistics[customStatistics] = comparands - .Select(comparand => calculation(comparand.Statistics) ?? double.NaN) - .ToArray(); - } + Baselines = baselines ?? new List(); + Comparands = comparands ?? new List(); } public List ComparisonResults { get; set; } @@ -120,13 +57,17 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline public string ComparandRunName { get; } public string ComparisonName => $"{ComparandRunName} vs {BaselineRunName}"; public string MicrobenchmarkName { get; } - public double[] OriginalBaselineMeanValueCollection { get; } - public double[] OriginalComparandMeanValueCollection { get; } + public IEnumerable Baselines { get; } + public IEnumerable Comparands { get; } public double[] OutliersFreeBaselineMeanValueCollection => - GC.Analysis.API.Statistics.RemoveOutliers(OriginalBaselineMeanValueCollection).ToArray(); + API.Statistics.RemoveOutliers(Baselines + .Select(baseline => baseline.Statistics?.Mean ?? double.NaN)) + .ToArray(); public double[] OutliersFreeComparandMeanValueCollection => - GC.Analysis.API.Statistics.RemoveOutliers(OriginalComparandMeanValueCollection).ToArray(); + API.Statistics.RemoveOutliers(Comparands + .Select(comparand => comparand.Statistics?.Mean ?? double.NaN)) + .ToArray(); public double AveragedBaselineMeanValue => API.GoodLinq.Average(OutliersFreeBaselineMeanValueCollection, r => r); public double AveragedComparandMeanValue => API.GoodLinq.Average(OutliersFreeComparandMeanValueCollection, r => r); @@ -150,73 +91,31 @@ public double MeanDiffPerc{ } } - public Dictionary OriginalBaselineCustomStatistics { get; } = new(); - public Dictionary OriginalComparandCustomStatistics { get; } = new(); - public Dictionary OutliersFreeBaselineCustomStatistics => OriginalBaselineCustomStatistics + public Dictionary OriginalBaselineOtherMetrics { get; } = new(); + public Dictionary OriginalComparandOtherMetrics { get; } = new(); + public Dictionary OutliersFreeBaselineOtherMetrics => OriginalBaselineOtherMetrics .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) .ToDictionary(); - public Dictionary OutliersFreeComparandCustomStatistics => OriginalComparandCustomStatistics + public Dictionary OutliersFreeComparandOtherMetrics => OriginalComparandOtherMetrics .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) .ToDictionary(); - public Dictionary AveragedBaselineCustomStatistics => OutliersFreeBaselineCustomStatistics + public Dictionary AveragedBaselineOtherMetrics => OutliersFreeBaselineOtherMetrics .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) .ToDictionary(); - public Dictionary AveragedComparandCustomStatistics => OutliersFreeComparandCustomStatistics + public Dictionary AveragedComparandOtherMetrics => OutliersFreeComparandOtherMetrics .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) .ToDictionary(); - public Dictionary CustomStatisticsDiff => OutliersFreeBaselineCustomStatistics - .Select(kvp => (kvp.Key, AveragedComparandCustomStatistics[kvp.Key] - AveragedBaselineCustomStatistics[kvp.Key])) - .ToDictionary(); - - public Dictionary CustomStatisticsDiffPerc => OutliersFreeBaselineCustomStatistics - .Select(kvp => - { - if (AveragedBaselineCustomStatistics[kvp.Key] == 0) - { - if (AveragedComparandCustomStatistics[kvp.Key] == 0) - { - return (kvp.Key, 0); - } - else - { - return (kvp.Key, double.NaN); - } - } - return (kvp.Key, CustomStatisticsDiff[kvp.Key] / AveragedBaselineCustomStatistics[kvp.Key]); - }) - .ToDictionary(); - - public Dictionary OriginalBaselineCustomGCData { get; } = new(); - public Dictionary OriginalComparandCustomGCData { get; } = new(); - public Dictionary OutliersFreeBaselineCustomGCData => OriginalBaselineCustomGCData - .Select(kvp => - (kvp.Key, GC.Analysis.API.Statistics.RemoveOutliers(kvp.Value).ToArray())) - .ToDictionary(); - public Dictionary OutliersFreeComparandCustomGCData => OriginalComparandCustomGCData - .Select(kvp => - (kvp.Key, GC.Analysis.API.Statistics.RemoveOutliers(kvp.Value).ToArray())) - .ToDictionary(); - public Dictionary AveragedBaselineCustomGCData => OutliersFreeBaselineCustomGCData - .Select(kvp => - (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) - .ToDictionary(); - public Dictionary AveragedComparandCustomGCData => OutliersFreeComparandCustomGCData - .Select(kvp => - (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) - .ToDictionary(); - - public Dictionary CustomGCDataDiff => OutliersFreeBaselineCustomGCData - .Select(kvp => - (kvp.Key, AveragedComparandCustomGCData[kvp.Key] - AveragedBaselineCustomGCData[kvp.Key])) + public Dictionary OtherMetricsDiff => OutliersFreeBaselineOtherMetrics + .Select(kvp => (kvp.Key, AveragedComparandOtherMetrics[kvp.Key] - AveragedBaselineOtherMetrics[kvp.Key])) .ToDictionary(); - public Dictionary CustomGCDataDiffPerc => OutliersFreeBaselineCustomGCData + public Dictionary OtherMetricsDiffPerc => OutliersFreeBaselineOtherMetrics .Select(kvp => { - if (AveragedBaselineCustomGCData[kvp.Key] == 0) + if (AveragedBaselineOtherMetrics[kvp.Key] == 0) { - if (AveragedComparandCustomGCData[kvp.Key] == 0) + if (AveragedComparandOtherMetrics[kvp.Key] == 0) { return (kvp.Key, 0); } @@ -225,7 +124,7 @@ public double MeanDiffPerc{ return (kvp.Key, double.NaN); } } - return (kvp.Key, CustomGCDataDiff[kvp.Key] / AveragedBaselineCustomGCData[kvp.Key]); + return (kvp.Key, OtherMetricsDiff[kvp.Key] / AveragedBaselineOtherMetrics[kvp.Key]); }) .ToDictionary(); } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs index 2763dbfb6a1..d64cdf9566f 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -1,16 +1,88 @@ -using GC.Analysis.API; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; +using API = GC.Analysis.API; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { public sealed class MicrobenchmarkResult { - public Statistics? Statistics { get; set; } - public GCProcessData? GCData { get; set; } + public static readonly IReadOnlyDictionary> CustomStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "number of iterations", (Statistics stats) => stats.N }, + { "min", (Statistics stats) => stats.Min }, + { "max", (Statistics stats) => stats.Max }, + { "median", (Statistics stats) => stats.Median }, + { "q1", (Statistics stats) => stats.Q1 }, + { "q3", (Statistics stats) => stats.Q3 }, + { "variance", (Statistics stats) => stats.Variance }, + { "standard deviation", (Statistics stats) => stats.StandardDeviation }, + { "skewness", (Statistics stats) => stats.Skewness }, + { "kurtosis", (Statistics stats) => stats.Kurtosis }, + { "standard error", (Statistics stats) => stats.StandardError }, + { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, + }; + + public static readonly Dictionary> CustomAggregateCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "gc count", (gc) => gc.Stats.Count }, + { "non induced gc count", (gc) => gc.Stats.Count - gc.GCs.Count(g => g.Reason == GCReason.Induced)}, + { "induced gc count", (gc) => gc.GCs.Count(g => g.Reason == GCReason.Induced)}, + { "total allocated (mb)", (gc) => gc.Stats.TotalAllocatedMB }, + { "max size peak (mb)", (gc) => gc.Stats.MaxSizePeakMB }, + { "total pause time (msec)", (gc) => gc.Stats.TotalPauseTimeMSec }, + { "gc pause time %", (gc) => gc.Stats.GetGCPauseTimePercentage() }, + { "avg. heap size (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeBeforeMB) }, + { "avg. heap size after (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeAfterMB) }, + }; + + public MicrobenchmarkResult(string benchmarkFullName, + Run parent, + Benchmark benchmark, + API.GCProcessData? gcData = null, + GCTraceMetrics? gcTraceMetrics = null, + API.CPUProcessData? cpuData = null, + IEnumerable? additionalReportMetrics = null, + IEnumerable? columns = null, + IEnumerable? cpuColumns = null) + { + MicrobenchmarkName = benchmarkFullName; + Parent = parent; + Statistics = benchmark.Statistics; + GCTraceMetrics = gcTraceMetrics; + CPUData = cpuData; + + if (additionalReportMetrics != null) + { + OtherMetrics = benchmark.Metrics + .Where(metric => additionalReportMetrics.Contains(metric.Descriptor.Id)) + .ToDictionary(metric => metric.Descriptor.Id, metric => (double?)metric.Value); + } + + if (columns != null) + { + var customStatistics = columns + .Where(column => CustomStatisticsCalculationMap.Keys.Contains(column)) + .Select(column => (column, CustomStatisticsCalculationMap[column](benchmark.Statistics))) + .ToDictionary(); + + OtherMetrics = OtherMetrics.Concat(customStatistics).ToDictionary(); + + if (gcData != null) + { + var customGCData = columns + .Where(column => CustomAggregateCalculationMap.Keys.Contains(column)) + .Select(column => (column, (double?)CustomAggregateCalculationMap[column](gcData))) + .ToDictionary(); + + OtherMetrics = OtherMetrics.Concat(customGCData).ToDictionary(); + } + } + } + public string MicrobenchmarkName { get; set; } + public Run Parent { get; set; } + public Statistics Statistics { get; set; } public GCTraceMetrics? GCTraceMetrics { get; set; } - public CPUProcessData? CPUData { get; set; } - public Run? Parent { get; set; } - public string? MicrobenchmarkName { get; set; } + public Dictionary OtherMetrics { get; set; } = new(); + public API.CPUProcessData? CPUData { get; set; } } - } \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 0a27bc2786b..746c9589521 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -40,16 +40,6 @@ public static class MicrobenchmarkResultComparison { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_10000_"}, }; - public static readonly Dictionary> DiffLevelPredicatorMap = new() - { - { "Large Regressions (>=20%)", r => r.MeanDiffPerc >= 20 }, - { "Large Improvements (<=-20%)", r => r.MeanDiffPerc <= -20 }, - { "Regressions (5% - 20%)", r => r.MeanDiffPerc >= 5 && r.MeanDiffPerc < 20 }, - { "Improvements (-20% - -5%)", r => r.MeanDiffPerc <= -5 && r.MeanDiffPerc > -20 }, - { "Stale Regressions (0% - 5%)", r => r.MeanDiffPerc >= 0 && r.MeanDiffPerc < 5 }, - { "Stale Improvements (0% - -5%)", r => r.MeanDiffPerc <= 0 && r.MeanDiffPerc > -5 }, - }; - private static readonly ConcurrentDictionary>> _benchmarkFullNameToJsonForRun = new(); public static ConcurrentDictionary> MapBenchmarkFullNameToJsonForRun(string outputPathForRun) @@ -146,56 +136,68 @@ public static IReadOnlyDictionary> Anal { Statistics statistics = benchmark.Statistics; - MicrobenchmarkResult microbenchmarkResult = new() - { - Statistics = statistics, - Parent = run.Value, - MicrobenchmarkName = benchmarkFullName - }; - + MicrobenchmarkResult? microbenchmarkResult = null; if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") { var jsonTraceMap = MapJsonToTraceForSingleBenchmarkRun(outputPathForRun, benchmarkFullName); string tracePath = jsonTraceMap.GetValueOrDefault(jsonPath, ""); - using var analyzer = AnalyzerManager.GetAnalyzer(tracePath); - List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); - List corerunProcesses = analyzer.GetProcessGCData("corerun"); - allPertinentProcesses.AddRange(corerunProcesses); - - GCProcessData? benchmarkGCData = null; - foreach (var process in allPertinentProcesses) + using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) { - string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); - string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); - if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) + List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); + List corerunProcesses = analyzer.GetProcessGCData("corerun"); + allPertinentProcesses.AddRange(corerunProcesses); + + GCProcessData? benchmarkGCData = null; + foreach (var process in allPertinentProcesses) { - benchmarkGCData = process; - break; + string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); + string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); + if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) + { + benchmarkGCData = process; + break; + } } - } - - if (benchmarkGCData != null) - { - int processID = benchmarkGCData.ProcessID; - microbenchmarkResult.GCData = benchmarkGCData; - microbenchmarkResult.GCTraceMetrics = new GCTraceMetrics(benchmarkGCData, tracePath, benchmark.FullName); - /* - TODO: THIS NEEDS TO BE ADDED BACK. - if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) + if (benchmarkGCData != null) { - // TODO: Add parameterize. - benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", - symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), - symbolPath: Path.Combine(configuration.Output.Path, run.Key)); - var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); - d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); - benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); + int processID = benchmarkGCData.ProcessID; + + /* + TODO: THIS NEEDS TO BE ADDED BACK. + if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) + { + // TODO: Add parameterize. + benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", + symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), + symbolPath: Path.Combine(configuration.Output.Path, run.Key)); + var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); + d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); + benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); + } + */ + microbenchmarkResult = new(benchmarkFullName, + run.Value, + benchmark, + gcData: benchmarkGCData, + gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, tracePath, benchmark.FullName), + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); } - */ } + System.GC.Collect(2); + } + else + { + microbenchmarkResult = new(benchmarkFullName, + run.Value, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); } - runsToResults[run.Value].Add(microbenchmarkResult); + runsToResults[run.Value].Add(microbenchmarkResult!); } }); diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index 03eace3be9c..73f25eab143 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -84,50 +84,44 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben sw.AddTableForSingleCriteria(configuration, comparisonResult.StaleImprovements); sw.WriteLine("\n\n"); - //if (configuration.Output.additional_report_metrics != null) - //{ - // foreach (var metric in configuration.Output.additional_report_metrics) - // { - // sw.WriteLine($"## Comparison by {metric}"); - // var ordered = comparisonResult.Comparisons.OrderByDescending(c => c.GetDiffPercentFromOtherMetrics(metric)); - - // // Large Regressions - // sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); - // sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.2)); - // sw.WriteLine("\n"); - - // // Large Improvements - // sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); - // var largeImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < -0.2); - // largeImprovements.Reverse(); - // sw.AddTableForSingleCriteria(configuration, largeImprovements); - // sw.WriteLine("\n"); - - // // Regressions - // sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); - // sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2)); - // sw.WriteLine("\n"); - - // // Improvements - // sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); - // var improvements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2); - // improvements.Reverse(); - // sw.AddTableForSingleCriteria(configuration, improvements); - // sw.WriteLine("\n"); - - // // Stale Regressions - // sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); - // sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < 0.05 && o.GetDiffPercentFromOtherMetrics(metric) >= 0.0)); - // sw.WriteLine("\n"); - - // // Stale Improvements - // sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); - // var staleImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > -0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.0); - // staleImprovements.Reverse(); - // sw.AddTableForSingleCriteria(configuration, staleImprovements); - // sw.WriteLine("\n"); - // } - //} + if (configuration.Output.additional_report_metrics != null) + { + foreach (var metric in configuration.Output.additional_report_metrics) + { + sw.WriteLine($"## Comparison by {metric}"); + var ordered = comparisonResult.Comparisons.OrderByDescending(c => c.OtherMetricsDiffPerc[metric]); + + // Large Regressions + sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.2)); + sw.WriteLine("\n"); + + // Large Improvements + sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] < -0.2)); + sw.WriteLine("\n"); + + // Regressions + sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); + sw.WriteLine("\n"); + + // Improvements + sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); + sw.WriteLine("\n"); + + // Stale Regressions + sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] < 0.05 && o.OtherMetricsDiffPerc[metric] >= 0.0)); + sw.WriteLine("\n"); + + // Stale Improvements + sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -0.05 && o.OtherMetricsDiffPerc[metric] < 0.0)); + sw.WriteLine("\n"); + } + } } internal static void AddTableForSingleCriteria(this StreamWriter sw, MicrobenchmarkConfiguration configuration, IEnumerable comparisons) @@ -168,19 +162,9 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm { foreach (var column in configuration.Output.Columns) { - double? baselineValue = null; - double? comparandValue = null; - if (MicrobenchmarkComparisonResult.CustomStatisticsCalculationMap.Keys.Contains(column)) - { - baselineValue = lr.AveragedBaselineCustomStatistics.GetValueOrDefault(column); - comparandValue = lr.AveragedComparandCustomStatistics.GetValueOrDefault(column); - } - if (MicrobenchmarkComparisonResult.CustomAggregateCalculationMap.Keys.Contains(column)) - { - baselineValue = lr.AveragedBaselineCustomGCData.GetValueOrDefault(column); - comparandValue = lr.AveragedComparandCustomGCData.GetValueOrDefault(column); - } - + double? baselineValue = lr.AveragedBaselineOtherMetrics.GetValueOrDefault(column); + double? comparandValue = lr.AveragedComparandOtherMetrics.GetValueOrDefault(column); + string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 4).ToString() : string.Empty; string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 4).ToString() : string.Empty; double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index af0af6f7da2..b07cde1f428 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -1,5 +1,4 @@ -using GC.Infrastructure.Core.Analysis; -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.Presentation.Microbenchmarks; @@ -41,12 +40,22 @@ public static List ExecuteAnalysis(Microbenchma var benchmarkFullNameJsonMap = MicrobenchmarkResultComparison.MapBenchmarkFullNameToJsonForRun(outputPathForRun); List comparisonResultForAllBenchmarks = new(); - foreach (var benchmarkFullName in benchmarkFullNameJsonMap.Keys) + ParallelOptions options = new ParallelOptions + { + MaxDegreeOfParallelism = System.Environment.ProcessorCount + }; + + object _lock = new(); + + Parallel.ForEach(benchmarkFullNameJsonMap.Keys, options, benchmarkFullName => { - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Analyzing Microbenchmarks: {benchmarkFullName} [/]\n"); List comparisonResultsForBenchmark = MicrobenchmarkResultComparison.CompareMicrobenchmarkResultForBenchmark(configuration, benchmarkFullName); - comparisonResultForAllBenchmarks.AddRange(comparisonResultsForBenchmark); - } + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Analysis For Microbenchmarks: {benchmarkFullName} completed. [/]\n"); + lock (_lock) + { + comparisonResultForAllBenchmarks.AddRange(comparisonResultsForBenchmark); + } + }); return MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResultForAllBenchmarks); } From 3b78ca34a767cc1e639f63a0024839cb858b43b3 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 14 May 2026 15:08:10 +0800 Subject: [PATCH 22/54] redesign microbenchmarkresult --- .../Analysis/BdnJsonResult.cs | 44 +-- .../MicrobenchmarkComparisonResult.cs | 18 + .../Microbenchmarks/MicrobenchmarkResult.cs | 8 +- .../MicrobenchmarkResultComparison.cs | 313 +++++++++--------- .../MicrobenchmarkAnalyzeCommand.cs | 35 +- 5 files changed, 216 insertions(+), 202 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs index e228ca6eff4..a64f11000a4 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs @@ -59,12 +59,12 @@ public sealed class Descriptor public sealed class ConfidenceInterval { public int N { get; set; } - public double? Mean { get; set; } - public double? StandardError { get; set; } - public int? Level { get; set; } - public double? Margin { get; set; } - public double? Lower { get; set; } - public double? Upper { get; set; } + public double Mean { get; set; } + public double StandardError { get; set; } + public int Level { get; set; } + public double Margin { get; set; } + public double Lower { get; set; } + public double Upper { get; set; } } public sealed class Percentiles @@ -84,23 +84,23 @@ public sealed class Statistics { public List OriginalValues { get; set; } public int N { get; set; } - public double? Min { get; set; } - public double? LowerFence { get; set; } - public double? Q1 { get; set; } - public double? Median { get; set; } - public double? Mean { get; set; } - public double? Q3 { get; set; } - public double? UpperFence { get; set; } - public double? Max { get; set; } - public double? InterquartileRange { get; set; } - public List LowerOutliers { get; set; } + public double Min { get; set; } + public double LowerFence { get; set; } + public double Q1 { get; set; } + public double Median { get; set; } + public double Mean { get; set; } + public double Q3 { get; set; } + public double UpperFence { get; set; } + public double Max { get; set; } + public double InterquartileRange { get; set; } + public List LowerOutliers { get; set; } public List UpperOutliers { get; set; } - public List AllOutliers { get; set; } - public double? StandardError { get; set; } - public double? Variance { get; set; } - public double? StandardDeviation { get; set; } - public double? Skewness { get; set; } - public double? Kurtosis { get; set; } + public List AllOutliers { get; set; } + public double StandardError { get; set; } + public double Variance { get; set; } + public double StandardDeviation { get; set; } + public double Skewness { get; set; } + public double Kurtosis { get; set; } public ConfidenceInterval? ConfidenceInterval { get; set; } public Percentiles Percentiles { get; set; } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index ade87c6d887..5babd4d43be 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -50,6 +50,24 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline Baselines = baselines ?? new List(); Comparands = comparands ?? new List(); + + foreach (var baseline in Baselines) + { + foreach (var kvp in baseline.OtherMetrics) + { + OriginalBaselineOtherMetrics[kvp.Key] = OriginalBaselineOtherMetrics.GetValueOrDefault(kvp.Key, Array.Empty()); + OriginalBaselineOtherMetrics[kvp.Key] = OriginalBaselineOtherMetrics[kvp.Key].Append(kvp.Value).ToArray(); + } + } + + foreach (var comparand in Comparands) + { + foreach (var kvp in comparand.OtherMetrics) + { + OriginalComparandOtherMetrics[kvp.Key] = OriginalComparandOtherMetrics.GetValueOrDefault(kvp.Key, Array.Empty()); + OriginalComparandOtherMetrics[kvp.Key] = OriginalComparandOtherMetrics[kvp.Key].Append(kvp.Value).ToArray(); + } + } } public List ComparisonResults { get; set; } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs index d64cdf9566f..15cb9dd91f4 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -6,7 +6,7 @@ namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { public sealed class MicrobenchmarkResult { - public static readonly IReadOnlyDictionary> CustomStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + public static readonly IReadOnlyDictionary> CustomStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) { { "number of iterations", (Statistics stats) => stats.N }, { "min", (Statistics stats) => stats.Min }, @@ -55,7 +55,7 @@ public MicrobenchmarkResult(string benchmarkFullName, { OtherMetrics = benchmark.Metrics .Where(metric => additionalReportMetrics.Contains(metric.Descriptor.Id)) - .ToDictionary(metric => metric.Descriptor.Id, metric => (double?)metric.Value); + .ToDictionary(metric => metric.Descriptor.Id, metric => metric.Value); } if (columns != null) @@ -71,7 +71,7 @@ public MicrobenchmarkResult(string benchmarkFullName, { var customGCData = columns .Where(column => CustomAggregateCalculationMap.Keys.Contains(column)) - .Select(column => (column, (double?)CustomAggregateCalculationMap[column](gcData))) + .Select(column => (column, CustomAggregateCalculationMap[column](gcData))) .ToDictionary(); OtherMetrics = OtherMetrics.Concat(customGCData).ToDictionary(); @@ -82,7 +82,7 @@ public MicrobenchmarkResult(string benchmarkFullName, public Run Parent { get; set; } public Statistics Statistics { get; set; } public GCTraceMetrics? GCTraceMetrics { get; set; } - public Dictionary OtherMetrics { get; set; } = new(); + public Dictionary OtherMetrics { get; set; } = new(); public API.CPUProcessData? CPUData { get; set; } } } \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 746c9589521..27bda97af32 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -1,4 +1,5 @@ using GC.Analysis.API; +using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; using Newtonsoft.Json; using System.Collections.Concurrent; @@ -7,6 +8,7 @@ namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { public static class MicrobenchmarkResultComparison { + private static readonly int _CPUCount = System.Environment.ProcessorCount; private static readonly Dictionary _benchmarkNameToTraceFilePatternMap = new() { { "ByteMark.BenchBitOps", "ByteMark.BenchBitOps"}, @@ -40,206 +42,221 @@ public static class MicrobenchmarkResultComparison { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_10000_"}, }; - private static readonly ConcurrentDictionary>> _benchmarkFullNameToJsonForRun = new(); - - public static ConcurrentDictionary> MapBenchmarkFullNameToJsonForRun(string outputPathForRun) + public static ConcurrentBag> LoadBdnJsonResults(MicrobenchmarkConfiguration configuration) { - return _benchmarkFullNameToJsonForRun.GetOrAdd(outputPathForRun, path => + ConcurrentBag> bdnJsonResults = new(); + Parallel.ForEach(configuration.Runs, (run) => { - ConcurrentDictionary> benchmarkFullNameJsonMap = new(); - + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); - - Parallel.ForEach(jsonFiles, (jsonFile) => + Parallel.ForEach(jsonFiles, jsonPath => { - BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); - string? fullName = results?.Benchmarks?.FirstOrDefault()?.FullName; - if (fullName != null) + run.Value.Name ??= run.Key; + BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); + if (results != null) { - benchmarkFullNameJsonMap.GetOrAdd(fullName, _ => new ConcurrentBag()).Add(jsonFile); + bdnJsonResults.Add(new(run.Value, results, jsonPath)); } }); - - return benchmarkFullNameJsonMap; }); + return bdnJsonResults; } - public static ConcurrentDictionary MapJsonToTraceForSingleBenchmarkRun(string outputPathForRun, string benchmarkFullName) + public static Dictionary MapJsonToTrace(string outputPath, ConcurrentBag> bdnJsonResults) { - ConcurrentDictionary jsonTraceMap = new(); - - var benchmarkFullNameJsonMap = MapBenchmarkFullNameToJsonForRun(outputPathForRun); - - string[] jsonFiles = benchmarkFullNameJsonMap.GetValueOrDefault(benchmarkFullName, new()).ToArray(); - - Parallel.ForEach(jsonFiles, (jsonFile) => + Dictionary jsonToTrace = new(); + foreach (var groupForRun in bdnJsonResults.GroupBy(t => t.Item1)) { - // placeholder - jsonTraceMap[jsonFile] = ""; - }); + var run = groupForRun.Key; - string[] sortedJsonFiles = jsonTraceMap.Keys - .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) - .ToArray(); + // GroupBy(t => t.Item2.Benchmarks.First().FullName) is not a bug: + // In single *full.json, multiple benchmarks stands for multiple input parameter combinations for the same benchmark + // If trace collection is enabled, process data for all those parameter combinations will be in the same trace file + foreach (var g in groupForRun.GroupBy(t => t.Item2.Benchmarks.First().FullName)) + { + var benchmarkName = g.Key; + var sortedJsonFiles = GoodLinq.Select(g, t => t.Item3) + .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) + .ToArray(); + + var traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkName]; + string outputPathForRun = Path.Combine(outputPath, run.Name); + var sortedTraceFiles = Directory.GetFiles(outputPathForRun, $"{traceFileNameTemplate}*.etl.zip", SearchOption.TopDirectoryOnly) + .OrderBy(traceFile => traceFile) + .ToArray(); + + if (sortedJsonFiles.Length != sortedTraceFiles.Length) + { + throw new InvalidOperationException( + $"The number of JSON files ({sortedJsonFiles.Length}) does not match the number of trace files ({sortedTraceFiles.Length}) for benchmark: {benchmarkName}"); + } - if (!_benchmarkNameToTraceFilePatternMap.Keys.Contains(benchmarkFullName)) - { - throw new KeyNotFoundException("No trace file pattern found for benchmark: " + benchmarkFullName); + for (int i = 0; i < sortedJsonFiles.Length; i++) + { + jsonToTrace[sortedJsonFiles[i]] = sortedTraceFiles[i]; + } + } } - string traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkFullName]; + - string[] sortedTraceFiles = Enumerable.Where(Directory.GetFiles(outputPathForRun, "*.etl.zip", SearchOption.TopDirectoryOnly), traceFile => - Path.GetFileName(traceFile).ToLower().Contains(traceFileNameTemplate.ToLower())) - .OrderBy(traceFile => traceFile) - .ToArray(); + return jsonToTrace; + } - if (sortedJsonFiles.Length != sortedTraceFiles.Length) - { - throw new InvalidOperationException( - $"The number of JSON files ({sortedJsonFiles.Length}) does not match the number of trace files ({sortedTraceFiles.Length}) for benchmark: {benchmarkFullName}"); - } + public static ConcurrentBag + AnalyzeMicrobenchmarkResults(MicrobenchmarkConfiguration configuration, + ConcurrentBag> bdnJsonResults, + bool excludeTraces = false) + { + ConcurrentBag microbenchmarkResults = new(); - for (int idx = 0; idx < sortedJsonFiles.Length; idx++) + Dictionary jsonToTraceMap = new(); + if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") { - jsonTraceMap[sortedJsonFiles[idx]] = sortedTraceFiles[idx]; + jsonToTraceMap = MapJsonToTrace(configuration.Output.Path, bdnJsonResults); } + + ParallelOptions options = new() + { + MaxDegreeOfParallelism = _CPUCount + }; - return jsonTraceMap; - } - - public static IReadOnlyDictionary> AnalyzeMicrobenchmarkResultsForSingleBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) - { - ConcurrentDictionary> runsToResults = new(); + int count = 0; + object _lock = new(); - Parallel.ForEach(configuration.Runs, (run) => + Parallel.ForEach(bdnJsonResults, options, t => { - string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); - run.Value.Name ??= run.Key; + var run = t.Item1; + var bdnJsonResult = t.Item2; + var jsonPath = t.Item3; - var benchmarkToJsonMapForRun = MapBenchmarkFullNameToJsonForRun(outputPathForRun); - var jsonFiles = benchmarkToJsonMapForRun.GetValueOrDefault(benchmarkFullName, new()); + List? benchmarks = bdnJsonResult?.Benchmarks; - runsToResults[run.Value] = runsToResults.GetValueOrDefault(run.Value, new()); - - Parallel.ForEach(jsonFiles, jsonPath => + if (benchmarks == null) { - BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); - - List? benchmarks = results?.Benchmarks; - - if (benchmarks == null) - { - return; - } + return; + } - foreach (var benchmark in benchmarks) + foreach (var benchmark in benchmarks) + { + Statistics statistics = benchmark.Statistics; + var benchmarkFullName = benchmark.FullName; + MicrobenchmarkResult? microbenchmarkResult = null; + if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") { - Statistics statistics = benchmark.Statistics; + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name!); + string tracePath = jsonToTraceMap.GetValueOrDefault(jsonPath, ""); - MicrobenchmarkResult? microbenchmarkResult = null; - if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") + using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) { - var jsonTraceMap = MapJsonToTraceForSingleBenchmarkRun(outputPathForRun, benchmarkFullName); - string tracePath = jsonTraceMap.GetValueOrDefault(jsonPath, ""); + List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); + List corerunProcesses = analyzer.GetProcessGCData("corerun"); + allPertinentProcesses.AddRange(corerunProcesses); - using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) + GCProcessData? benchmarkGCData = null; + foreach (var process in allPertinentProcesses) { - List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); - List corerunProcesses = analyzer.GetProcessGCData("corerun"); - allPertinentProcesses.AddRange(corerunProcesses); - - GCProcessData? benchmarkGCData = null; - foreach (var process in allPertinentProcesses) + string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); + string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); + if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) { - string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); - string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); - if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) - { - benchmarkGCData = process; - break; - } + benchmarkGCData = process; + break; } - if (benchmarkGCData != null) - { - int processID = benchmarkGCData.ProcessID; + } + if (benchmarkGCData != null) + { + int processID = benchmarkGCData.ProcessID; - /* - TODO: THIS NEEDS TO BE ADDED BACK. - if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) - { - // TODO: Add parameterize. - benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", - symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), - symbolPath: Path.Combine(configuration.Output.Path, run.Key)); - var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); - d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); - benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); - } - */ - microbenchmarkResult = new(benchmarkFullName, - run.Value, - benchmark, - gcData: benchmarkGCData, - gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, tracePath, benchmark.FullName), - additionalReportMetrics: configuration.Output.additional_report_metrics, - cpuColumns: configuration.Output.cpu_columns, - columns: configuration.Output.Columns); + /* + TODO: THIS NEEDS TO BE ADDED BACK. + if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) + { + // TODO: Add parameterize. + benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", + symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), + symbolPath: Path.Combine(configuration.Output.Path, run.Key)); + var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); + d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); + benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); } + */ + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + gcData: benchmarkGCData, + gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, tracePath, benchmark.FullName), + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); } - System.GC.Collect(2); } - else - { - microbenchmarkResult = new(benchmarkFullName, - run.Value, - benchmark, - additionalReportMetrics: configuration.Output.additional_report_metrics, - cpuColumns: configuration.Output.cpu_columns, - columns: configuration.Output.Columns); - } - runsToResults[run.Value].Add(microbenchmarkResult!); + System.GC.Collect(2); } - }); - + else + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + microbenchmarkResults.Add(microbenchmarkResult!); + } + lock (_lock) + { + count = count + 1; + Console.WriteLine($"\r{count}/{bdnJsonResults.Count} analyzed."); + } }); - return runsToResults; + return microbenchmarkResults; } - public static List CompareMicrobenchmarkResultForBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) + public static List CompareMicrobenchmarkResults(MicrobenchmarkConfiguration configuration, IEnumerable microbenchmarkResults, bool excludeTraces = false) { - bool includeTraces = (!excludeTraces) && configuration.TraceConfigurations.Type != "none"; - IReadOnlyDictionary> runResults = AnalyzeMicrobenchmarkResultsForSingleBenchmark(configuration, benchmarkFullName, excludeTraces); + bool includeTraces = (!excludeTraces) && (configuration.TraceConfigurations.Type != "none"); + var microbenchmarkResultsGroupedByBenchmarkName = microbenchmarkResults + .GroupBy(microbenchmarkResult => microbenchmarkResult.MicrobenchmarkName); + List comparisonResults = new(); - if (configuration.Output.run_comparisons != null) + object _lock = new(); + ParallelOptions options = new() + { + MaxDegreeOfParallelism = _CPUCount + }; + Parallel.ForEach(microbenchmarkResultsGroupedByBenchmarkName, options, microbenchmarkResultsGroup => { - foreach (var comparison in configuration.Output.run_comparisons) + if (configuration.Output.run_comparisons != null) { - string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); - string baselineName = breakup[0]; - string runName = breakup[1]; - - var baselineRuns = GoodLinq.Where(runResults.Keys, r => r.Name == baselineName); - var comparandRuns = GoodLinq.Where(runResults.Keys, r => r.Name == runName); + foreach (var comparison in configuration.Output.run_comparisons) + { + string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); + string baselineName = breakup[0]; + string runName = breakup[1]; - var baselineMicrobenchmarkResults = GoodLinq.Select(baselineRuns, b => runResults[b]).SelectMany(r => r); - var comparandMicrobenchmarkResults = GoodLinq.Select(comparandRuns, c => runResults[c]).SelectMany(r => r); + var baselineMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.Name == baselineName); + var comparandMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.Name == runName); - comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + lock (_lock) + { + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + } + } } - } - // Default case where the run comparisons aren't specified. - else - { - var baselineRuns = GoodLinq.Where(runResults.Keys, r => r.is_baseline); - var comparandRuns = GoodLinq.Where(runResults.Keys, r => !r.is_baseline); - - var baselineMicrobenchmarkResults = GoodLinq.Select(baselineRuns, b => runResults[b]).SelectMany(r => r); - var comparandMicrobenchmarkResults = GoodLinq.Select(comparandRuns, c => runResults[c]).SelectMany(r => r); + // Default case where the run comparisons aren't specified. + else + { + var baselineMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.is_baseline); + var comparandMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => !r.Parent.is_baseline); - comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); - } + lock (_lock) + { + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + } + } + }); return comparisonResults; } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index b07cde1f428..c9a9b521428 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -1,4 +1,5 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.Presentation.Microbenchmarks; @@ -31,33 +32,11 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben public static List ExecuteAnalysis(MicrobenchmarkConfiguration configuration) { - Run? run = configuration.Runs.Values.FirstOrDefault(); - if (run == null) - { - throw new InvalidOperationException("No runs found in the configuration."); - } - string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name); - var benchmarkFullNameJsonMap = MicrobenchmarkResultComparison.MapBenchmarkFullNameToJsonForRun(outputPathForRun); - List comparisonResultForAllBenchmarks = new(); - - ParallelOptions options = new ParallelOptions - { - MaxDegreeOfParallelism = System.Environment.ProcessorCount - }; - - object _lock = new(); - - Parallel.ForEach(benchmarkFullNameJsonMap.Keys, options, benchmarkFullName => - { - List comparisonResultsForBenchmark = MicrobenchmarkResultComparison.CompareMicrobenchmarkResultForBenchmark(configuration, benchmarkFullName); - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Analysis For Microbenchmarks: {benchmarkFullName} completed. [/]\n"); - lock (_lock) - { - comparisonResultForAllBenchmarks.AddRange(comparisonResultsForBenchmark); - } - }); - - return MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResultForAllBenchmarks); + var bdnJsonResults = MicrobenchmarkResultComparison.LoadBdnJsonResults(configuration); + var microbenchmarkResults = MicrobenchmarkResultComparison.AnalyzeMicrobenchmarkResults(configuration, bdnJsonResults); + var comparisonResults = MicrobenchmarkResultComparison.CompareMicrobenchmarkResults(configuration, microbenchmarkResults); + + return MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResults); } } } From 8096656e15793b775f84f8ab90d8d796db3f9d11 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 14 May 2026 16:33:22 +0800 Subject: [PATCH 23/54] move presentation to analyze-command --- .../MicrobenchmarkResultComparison.cs | 2 +- .../Microbenchmarks/Presentation.cs | 27 ----------------- .../MicrobenchmarkAnalyzeCommand.cs | 30 +++++++++++++++++-- .../Microbenchmark/MicrobenchmarkCommand.cs | 2 +- 4 files changed, 30 insertions(+), 31 deletions(-) delete mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 27bda97af32..2a14a91a128 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -206,7 +206,7 @@ public static ConcurrentBag lock (_lock) { count = count + 1; - Console.WriteLine($"\r{count}/{bdnJsonResults.Count} analyzed."); + Console.Write($"\r{count}/{bdnJsonResults.Count} microbenchmarks results analyzed."); } }); diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs deleted file mode 100644 index 8d879a48ecd..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs +++ /dev/null @@ -1,27 +0,0 @@ -using GC.Infrastructure.Core.Analysis; -using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; - -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks -{ - public static class Presentation - { - public static void Present(MicrobenchmarkConfiguration configuration, List comparisonResultsGroupedByName, Dictionary executionDetails) - { - foreach (var format in configuration.Output.Formats) - { - if (format == "markdown") - { - Markdown.GenerateTable(configuration, comparisonResultsGroupedByName, executionDetails, Path.Combine(configuration.Output.Path, "Results.md")); - continue; - } - - if (format == "json") - { - Json.Generate(configuration, comparisonResultsGroupedByName, Path.Combine(configuration.Output.Path, "Results.json")); - continue; - } - } - } - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index c9a9b521428..6f7e6d834b7 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -24,19 +24,45 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(MicrobenchmarkAnalyzeCommand)); MicrobenchmarkConfiguration configuration = MicrobenchmarkConfigurationParser.Parse(settings.ConfigurationPath); - var comparisonResultsGroupedName = ExecuteAnalysis(configuration); + var comparisonResultsGroupedByName = ExecuteAnalysis(configuration); - Presentation.Present(configuration, comparisonResultsGroupedName, new()); // Execution details aren't available for the analysis-only mode. + Present(configuration, comparisonResultsGroupedByName, new()); // Execution details aren't available for the analysis-only mode. return 0; } public static List ExecuteAnalysis(MicrobenchmarkConfiguration configuration) { var bdnJsonResults = MicrobenchmarkResultComparison.LoadBdnJsonResults(configuration); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) {bdnJsonResults.Count} BDN results loaded.[/]"); var microbenchmarkResults = MicrobenchmarkResultComparison.AnalyzeMicrobenchmarkResults(configuration, bdnJsonResults); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Analysis completed.[/]"); var comparisonResults = MicrobenchmarkResultComparison.CompareMicrobenchmarkResults(configuration, microbenchmarkResults); return MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResults); } + + public static void Present(MicrobenchmarkConfiguration configuration, + List comparisonResultsGroupedByName, + Dictionary executionDetails) + { + foreach (var format in configuration.Output.Formats) + { + if (format == "markdown") + { + string outputPath = Path.Combine(configuration.Output.Path, "Results.md"); + Markdown.GenerateTable(configuration, comparisonResultsGroupedByName, executionDetails, outputPath); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {outputPath}.[/]"); + continue; + } + + if (format == "json") + { + string outputPath = Path.Combine(configuration.Output.Path, "Results.json"); + Json.Generate(configuration, comparisonResultsGroupedByName, outputPath); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {outputPath}.[/]"); + continue; + } + } + } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index 74cb749e680..6bd1c3b9f73 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -202,7 +202,7 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi var comparisonResultsGroupedName = MicrobenchmarkAnalyzeCommand.ExecuteAnalysis(configuration); - Presentation.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. + MicrobenchmarkAnalyzeCommand.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. Directory.SetCurrentDirectory(currentDirectory); AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbechmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); return new MicrobenchmarkOutputResults(executionDetails, comparisonResultsGroupedName); From ec7b88204bb7f85ad0f2e48ef46be92ebc11fe48 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 15 May 2026 09:47:55 +0800 Subject: [PATCH 24/54] improve performance of analyzing stage --- .../MicrobenchmarkResultComparison.cs | 67 +++++++++++-------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 2a14a91a128..a4945f7187b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -116,7 +116,7 @@ public static ConcurrentBag } ParallelOptions options = new() - { + { MaxDegreeOfParallelism = _CPUCount }; @@ -136,22 +136,23 @@ public static ConcurrentBag return; } - foreach (var benchmark in benchmarks) + if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") { - Statistics statistics = benchmark.Statistics; - var benchmarkFullName = benchmark.FullName; - MicrobenchmarkResult? microbenchmarkResult = null; - if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name!); + string tracePath = jsonToTraceMap.GetValueOrDefault(jsonPath, ""); + + using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) { - string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name!); - string tracePath = jsonToTraceMap.GetValueOrDefault(jsonPath, ""); + List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); + List corerunProcesses = analyzer.GetProcessGCData("corerun"); + allPertinentProcesses.AddRange(corerunProcesses); - using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) + foreach (var benchmark in benchmarks) { - List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); - List corerunProcesses = analyzer.GetProcessGCData("corerun"); - allPertinentProcesses.AddRange(corerunProcesses); + Statistics statistics = benchmark.Statistics; + var benchmarkFullName = benchmark.FullName; + MicrobenchmarkResult? microbenchmarkResult = null; GCProcessData? benchmarkGCData = null; foreach (var process in allPertinentProcesses) { @@ -181,32 +182,40 @@ public static ConcurrentBag } */ microbenchmarkResult = new(benchmarkFullName, - run, - benchmark, - gcData: benchmarkGCData, - gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, tracePath, benchmark.FullName), - additionalReportMetrics: configuration.Output.additional_report_metrics, - cpuColumns: configuration.Output.cpu_columns, - columns: configuration.Output.Columns); + run, + benchmark, + gcData: benchmarkGCData, + gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, tracePath, benchmark.FullName), + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + microbenchmarkResults.Add(microbenchmarkResult!); } - } - System.GC.Collect(2); + } } - else + } + else + { + foreach (var benchmark in benchmarks) { + Statistics statistics = benchmark.Statistics; + var benchmarkFullName = benchmark.FullName; + + MicrobenchmarkResult? microbenchmarkResult = null; microbenchmarkResult = new(benchmarkFullName, - run, - benchmark, - additionalReportMetrics: configuration.Output.additional_report_metrics, - cpuColumns: configuration.Output.cpu_columns, - columns: configuration.Output.Columns); + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + microbenchmarkResults.Add(microbenchmarkResult!); } - microbenchmarkResults.Add(microbenchmarkResult!); } + lock (_lock) { count = count + 1; - Console.Write($"\r{count}/{bdnJsonResults.Count} microbenchmarks results analyzed."); + Console.Write($"\r{count}/{bdnJsonResults.Count} BDN results analyzed."); } }); From 6ba221940a80788ea790d0225c216e2e7f73a610 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 15 May 2026 14:09:58 +0800 Subject: [PATCH 25/54] includes int type properties --- .../Analysis/GCTraceMetrics.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs index df14a2368f9..d88777bc010 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs @@ -61,27 +61,29 @@ public GCTraceMetrics(GCProcessData processData, string runName, string configur var properties = processData.Stats.GetType().GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); foreach (var property in properties) { - if (property.PropertyType != typeof(double) || property.PropertyType != typeof(int)) + if (property.PropertyType != typeof(double) && property.PropertyType != typeof(int)) { continue; } string propertyName = property.Name; - double propertyValue = (double)(property.GetValue(processData.Stats) ?? double.NaN); + object? value = property.GetValue(processData.Stats); + double propertyValue = value != null ? Convert.ToDouble(value) : double.NaN; StatsData[propertyName] = propertyValue; } var fields = processData.Stats.GetType().GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); foreach (var field in fields) { - if (field.FieldType != typeof(double) || field.FieldType != typeof(int)) + if (field.FieldType != typeof(double) && field.FieldType != typeof(int)) { continue; } string name = field.Name; - double value = (double)(field.GetValue(processData.Stats) ?? double.NaN); - StatsData[name] = value; + object? value = field.GetValue(processData.Stats); + double doubleValue = value != null ? Convert.ToDouble(value) : double.NaN; + StatsData[name] = doubleValue; } // 95P From cc59167bb1ab0fb5e72def0a0477768050493ea9 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 15 May 2026 14:16:09 +0800 Subject: [PATCH 26/54] check key existence and set parallelism degree to 2 * cpu_count --- .../Microbenchmarks/MicrobenchmarkResultComparison.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index a4945f7187b..502ce6b5ece 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -79,6 +79,10 @@ public static Dictionary MapJsonToTrace(string outputPath, Concu .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) .ToArray(); + if (!_benchmarkNameToTraceFilePatternMap.ContainsKey(benchmarkName)) + { + throw new InvalidOperationException($"Benchmark name {benchmarkName} does not have a corresponding trace file pattern in the map."); + } var traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkName]; string outputPathForRun = Path.Combine(outputPath, run.Name); var sortedTraceFiles = Directory.GetFiles(outputPathForRun, $"{traceFileNameTemplate}*.etl.zip", SearchOption.TopDirectoryOnly) @@ -97,7 +101,6 @@ public static Dictionary MapJsonToTrace(string outputPath, Concu } } } - return jsonToTrace; } @@ -117,7 +120,7 @@ public static ConcurrentBag ParallelOptions options = new() { - MaxDegreeOfParallelism = _CPUCount + MaxDegreeOfParallelism = _CPUCount * 2 }; int count = 0; @@ -219,6 +222,7 @@ public static ConcurrentBag } }); + Console.WriteLine(); return microbenchmarkResults; } From a7c1b2c71a1be78e00cf3aae2654a2d7c252f2cb Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 15 May 2026 14:22:10 +0800 Subject: [PATCH 27/54] check if metricName is a key of StatsData --- .../Analysis/GCTraceMetricComparisonResult.cs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs index deb14e44080..9f749a6a0c7 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs @@ -43,15 +43,24 @@ public GCTraceMetricComparisonResult(IEnumerable baselines, IEnu else { - OriginalBaselineMetricCollection = baselines.Select(baseline => baseline.StatsData[fieldInfo.Name]); - OriginalComparandMetricCollection = comparands.Select(comparand => comparand.StatsData[fieldInfo.Name]); + OriginalBaselineMetricCollection = baselines + .Where(baseline => baseline.StatsData.ContainsKey(fieldInfo.Name)) + .Select(baseline => baseline.StatsData[fieldInfo.Name]); + + OriginalComparandMetricCollection = comparands + .Where(comparand => comparand.StatsData.ContainsKey(fieldInfo.Name)) + .Select(comparand => comparand.StatsData[fieldInfo.Name]); } } else { - OriginalBaselineMetricCollection = baselines.Select(baseline => baseline.StatsData[pInfo.Name]); - OriginalComparandMetricCollection = comparands.Select(comparand => comparand.StatsData[pInfo.Name]); + OriginalBaselineMetricCollection = baselines + .Where(baseline => baseline.StatsData.ContainsKey(pInfo.Name)) + .Select(baseline => baseline.StatsData[pInfo.Name]); + OriginalComparandMetricCollection = comparands + .Where(comparand => comparand.StatsData.ContainsKey(pInfo.Name)) + .Select(comparand => comparand.StatsData[pInfo.Name]); } } From 357ce0a86e3959d0ee34b6869e92b0afadc484cb Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 15 May 2026 14:30:46 +0800 Subject: [PATCH 28/54] Filter out null GCTraceMetrics instances before calling CompareGCTraceMetric --- .../Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 5babd4d43be..c020bcb7fe9 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -24,11 +24,13 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline { var baselineGCTraceMetricsCollection = baselines .Where(baseline => baseline != null) + .Where(baseline => baseline.GCTraceMetrics != null) .Select(baseline => baseline.GCTraceMetrics) .ToArray(); var comparandGCTraceMetricsCollection = comparands .Where(comparand => comparand != null) + .Where(comparand => comparand.GCTraceMetrics != null) .Select(comparand => comparand.GCTraceMetrics) .ToArray(); From 3d1c330eb91205849b954f9c885ba071d5ab1c99 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 15 May 2026 15:12:07 +0800 Subject: [PATCH 29/54] update initialization of OtherMetrics for MicrobenchmarkComparisonResult --- .../MicrobenchmarkComparisonResult.cs | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index c020bcb7fe9..eb9945e621a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -53,23 +53,17 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline Baselines = baselines ?? new List(); Comparands = comparands ?? new List(); - foreach (var baseline in Baselines) - { - foreach (var kvp in baseline.OtherMetrics) - { - OriginalBaselineOtherMetrics[kvp.Key] = OriginalBaselineOtherMetrics.GetValueOrDefault(kvp.Key, Array.Empty()); - OriginalBaselineOtherMetrics[kvp.Key] = OriginalBaselineOtherMetrics[kvp.Key].Append(kvp.Value).ToArray(); - } - } - - foreach (var comparand in Comparands) - { - foreach (var kvp in comparand.OtherMetrics) - { - OriginalComparandOtherMetrics[kvp.Key] = OriginalComparandOtherMetrics.GetValueOrDefault(kvp.Key, Array.Empty()); - OriginalComparandOtherMetrics[kvp.Key] = OriginalComparandOtherMetrics[kvp.Key].Append(kvp.Value).ToArray(); - } - } + OriginalBaselineOtherMetrics = Baselines + .Select(baseline => baseline.OtherMetrics) + .SelectMany(kvp => kvp) + .GroupBy(kvp => kvp.Key) + .ToDictionary(g => g.Key, g => g.Select(kvp => kvp.Value).ToArray()); + + OriginalComparandOtherMetrics = Comparands + .Select(comparand => comparand.OtherMetrics) + .SelectMany(kvp => kvp) + .GroupBy(kvp => kvp.Key) + .ToDictionary(g => g.Key, g => g.Select(kvp => kvp.Value).ToArray()); } public List ComparisonResults { get; set; } From eb9f7e36436714530a366e5953c3a4e3b866c849 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 15 May 2026 15:28:21 +0800 Subject: [PATCH 30/54] comment out cpu_columns related code --- .../Presentation/Microbenchmarks/Markdown.cs | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index 73f25eab143..164a3cf4c22 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -93,22 +93,22 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben // Large Regressions sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.2)); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.2)); sw.WriteLine("\n"); // Large Improvements sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] < -0.2)); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.2)); sw.WriteLine("\n"); // Regressions sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); sw.WriteLine("\n"); // Improvements sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.05 && o.OtherMetricsDiffPerc[metric] > -0.2)); sw.WriteLine("\n"); // Stale Regressions @@ -118,7 +118,7 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben // Stale Improvements sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -0.05 && o.OtherMetricsDiffPerc[metric] < 0.0)); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -0.05 && o.OtherMetricsDiffPerc[metric] <= 0.0)); sw.WriteLine("\n"); } } @@ -139,14 +139,15 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm } } - if (configuration.Output.cpu_columns != null) - { - foreach (var column in configuration.Output.cpu_columns) - { - tableHeader0 += $"Baseline {column} | Comparand {column} | Δ {column} | Δ% {column} |"; - tableHeader1 += "--- | --- | --- | --- |"; - } - } + // TODO: Add CPU columns if needed in the future. + //if (configuration.Output.cpu_columns != null) + //{ + // foreach (var column in configuration.Output.cpu_columns) + // { + // tableHeader0 += $"Baseline {column} | Comparand {column} | Δ {column} | Δ% {column} |"; + // tableHeader1 += "--- | --- | --- | --- |"; + // } + //} sw.WriteLine(tableHeader0); sw.WriteLine(tableHeader1); @@ -177,6 +178,7 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm } } + // TODO: Add CPU columns if needed in the future. //if (configuration.Output.cpu_columns != null) //{ // foreach (var column in configuration.Output.cpu_columns) From 7d5709ae2944ba66911ee5ff6a4e0332f0d05f95 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 15 May 2026 17:27:38 +0800 Subject: [PATCH 31/54] remove unused imports --- .../Commands/Microbenchmark/MicrobenchmarkCommand.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index 6bd1c3b9f73..82901293291 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -4,13 +4,11 @@ using GC.Infrastructure.Core.CommandBuilders; using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using GC.Infrastructure.Core.Presentation.Microbenchmarks; using GC.Infrastructure.Core.TraceCollection; using Newtonsoft.Json; using Spectre.Console; using Spectre.Console.Cli; using System.ComponentModel; -using System.Configuration; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Text; From 994ea7abc6e1b54155862837f3ace6d692d15960 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 15 May 2026 17:28:37 +0800 Subject: [PATCH 32/54] "Microbechmark" (missing 'n') --- .../Commands/Microbenchmark/MicrobenchmarkCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index 82901293291..e411bbd2f65 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -202,7 +202,7 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi MicrobenchmarkAnalyzeCommand.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. Directory.SetCurrentDirectory(currentDirectory); - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbechmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbenchmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); return new MicrobenchmarkOutputResults(executionDetails, comparisonResultsGroupedName); } } From ac9ec4899896d25ae16f42f823070df8f9532fc7 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Fri, 15 May 2026 17:30:47 +0800 Subject: [PATCH 33/54] avoid breaking formatting --- .../Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index 6f7e6d834b7..a68217319c8 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -51,7 +51,7 @@ public static void Present(MicrobenchmarkConfiguration configuration, { string outputPath = Path.Combine(configuration.Output.Path, "Results.md"); Markdown.GenerateTable(configuration, comparisonResultsGroupedByName, executionDetails, outputPath); - AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {outputPath}.[/]"); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {Markup.Escape(outputPath)}.[/]"); continue; } From b7bfe3de79e4f2d763da9d058e8356a430cb80a5 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 09:54:51 +0800 Subject: [PATCH 34/54] sort improvements in ascending order --- .../MicrobenchmarkComparisonResults.cs | 2 +- .../Presentation/Microbenchmarks/Markdown.cs | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs index 79d2f8eccee..4a20b557394 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs @@ -21,7 +21,7 @@ public MicrobenchmarkComparisonResults(string baselineName, string runName, IEnu public IEnumerable LargeImprovements => Ordered.Where(o => o.MeanDiffPerc < -20).OrderBy(g => g.MeanDiffPerc); public IEnumerable Regressions => Ordered.Where(o => o.MeanDiffPerc < 20 && o.MeanDiffPerc > 5); public IEnumerable Improvements => Ordered.Where(o => o.MeanDiffPerc > -20 && o.MeanDiffPerc < -5).OrderBy(g => g.MeanDiffPerc); - public IEnumerable StaleRegressions => Ordered.Where((o => o.MeanDiffPerc > 0 && o.MeanDiffPerc < 5)).OrderByDescending(g => g.MeanDiffPerc); + public IEnumerable StaleRegressions => Ordered.Where((o => o.MeanDiffPerc > 0 && o.MeanDiffPerc < 5)); public IEnumerable StaleImprovements => Ordered.Where((o => o.MeanDiffPerc < 0 && o.MeanDiffPerc > -5)).OrderBy(g => g.MeanDiffPerc); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index 164a3cf4c22..17f03471a5c 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -98,7 +98,9 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben // Large Improvements sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.2)); + var largeImprovements = GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.2); + largeImprovements.Reverse(); + sw.AddTableForSingleCriteria(configuration, largeImprovements); sw.WriteLine("\n"); // Regressions @@ -108,7 +110,9 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben // Improvements sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.05 && o.OtherMetricsDiffPerc[metric] > -0.2)); + var improvements = GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.05 && o.OtherMetricsDiffPerc[metric] > -0.2); + improvements.Reverse(); + sw.AddTableForSingleCriteria(configuration, improvements); sw.WriteLine("\n"); // Stale Regressions @@ -118,7 +122,9 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben // Stale Improvements sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -0.05 && o.OtherMetricsDiffPerc[metric] <= 0.0)); + var staleImprovements = GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -0.05 && o.OtherMetricsDiffPerc[metric] <= 0.0); + staleImprovements.Reverse(); + sw.AddTableForSingleCriteria(configuration, staleImprovements); sw.WriteLine("\n"); } } From 08ec2a57e77a031c985d1185b2ba73dc3cc4d05d Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 10:01:23 +0800 Subject: [PATCH 35/54] Convert to ToDictionary(x => x.Item1, x => x.Item2) (or tuple names) for each projection --- .../MicrobenchmarkComparisonResult.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index eb9945e621a..a2c2e1dbfc4 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -109,20 +109,20 @@ public double MeanDiffPerc{ public Dictionary OriginalComparandOtherMetrics { get; } = new(); public Dictionary OutliersFreeBaselineOtherMetrics => OriginalBaselineOtherMetrics .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); public Dictionary OutliersFreeComparandOtherMetrics => OriginalComparandOtherMetrics .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); public Dictionary AveragedBaselineOtherMetrics => OutliersFreeBaselineOtherMetrics .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); public Dictionary AveragedComparandOtherMetrics => OutliersFreeComparandOtherMetrics .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); public Dictionary OtherMetricsDiff => OutliersFreeBaselineOtherMetrics .Select(kvp => (kvp.Key, AveragedComparandOtherMetrics[kvp.Key] - AveragedBaselineOtherMetrics[kvp.Key])) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); public Dictionary OtherMetricsDiffPerc => OutliersFreeBaselineOtherMetrics .Select(kvp => @@ -140,6 +140,6 @@ public double MeanDiffPerc{ } return (kvp.Key, OtherMetricsDiff[kvp.Key] / AveragedBaselineOtherMetrics[kvp.Key]); }) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); } } From 722ace7c2a7d64eb1f94dc0f6a5fc1621474cafd Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 10:09:53 +0800 Subject: [PATCH 36/54] provide key value pair projection --- .../Analysis/Microbenchmarks/MicrobenchmarkResult.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs index 15cb9dd91f4..f5fb0f898b3 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -63,18 +63,18 @@ public MicrobenchmarkResult(string benchmarkFullName, var customStatistics = columns .Where(column => CustomStatisticsCalculationMap.Keys.Contains(column)) .Select(column => (column, CustomStatisticsCalculationMap[column](benchmark.Statistics))) - .ToDictionary(); + .ToDictionary(x => x.column, x => x.Item2); - OtherMetrics = OtherMetrics.Concat(customStatistics).ToDictionary(); + OtherMetrics = OtherMetrics.Concat(customStatistics).ToDictionary(x => x.Key, x => x.Value); if (gcData != null) { var customGCData = columns .Where(column => CustomAggregateCalculationMap.Keys.Contains(column)) .Select(column => (column, CustomAggregateCalculationMap[column](gcData))) - .ToDictionary(); + .ToDictionary(x => x.column, x => x.Item2); - OtherMetrics = OtherMetrics.Concat(customGCData).ToDictionary(); + OtherMetrics = OtherMetrics.Concat(customGCData).ToDictionary(x => x.Key, x => x.Value); } } } From cf32db52392df3e55309d470bec88bd40edccb4d Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 14:50:34 +0800 Subject: [PATCH 37/54] sort in order by index --- .../Microbenchmarks/MicrobenchmarkResultComparison.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 502ce6b5ece..f684848044a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -3,6 +3,7 @@ using GC.Infrastructure.Core.Configurations.Microbenchmarks; using Newtonsoft.Json; using System.Collections.Concurrent; +using System.Text.RegularExpressions; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { @@ -62,6 +63,8 @@ public static ConcurrentBag> LoadBdnJsonResult return bdnJsonResults; } + // TODO: We should specify relationship between json files and trace files before running benchmarks instead of relying on file name patterns. + // This will make the mapping more robust and less prone to errors due to file naming. public static Dictionary MapJsonToTrace(string outputPath, ConcurrentBag> bdnJsonResults) { Dictionary jsonToTrace = new(); @@ -86,7 +89,11 @@ public static Dictionary MapJsonToTrace(string outputPath, Concu var traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkName]; string outputPathForRun = Path.Combine(outputPath, run.Name); var sortedTraceFiles = Directory.GetFiles(outputPathForRun, $"{traceFileNameTemplate}*.etl.zip", SearchOption.TopDirectoryOnly) - .OrderBy(traceFile => traceFile) + .OrderBy(traceFile => + { + var match = Regex.Match(Path.GetFileName(traceFile), @"_(\d+)\.etl\.zip$"); + return match.Success ? int.Parse(match.Groups[1].Value) : 0; + }) .ToArray(); if (sortedJsonFiles.Length != sortedTraceFiles.Length) From 766fc821a44c6553dfb0ca1e85f4a899837a6373 Mon Sep 17 00:00:00 2001 From: VincentBu <44959937+VincentBu@users.noreply.github.com> Date: Mon, 18 May 2026 15:05:49 +0800 Subject: [PATCH 38/54] remove Spectre markup tokens from output path Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index a68217319c8..b179894d524 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -59,7 +59,7 @@ public static void Present(MicrobenchmarkConfiguration configuration, { string outputPath = Path.Combine(configuration.Output.Path, "Results.json"); Json.Generate(configuration, comparisonResultsGroupedByName, outputPath); - AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {outputPath}.[/]"); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {Markup.Escape(outputPath)}.[/]"); continue; } } From 1efd8023c4ba284a76cf164603d3d302b88d2a0b Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 15:14:34 +0800 Subject: [PATCH 39/54] take non-replayable enumerables into consideration --- .../GC.Analysis.API/Statistics.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs index d0269d32326..609066f035a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs @@ -34,13 +34,22 @@ public static double StandardDeviation(this IEnumerable doubleList) public static IEnumerable RemoveOutliers(IEnumerable collection) { + List validCollection = new(); if (!collection.Any()) { return Array.Empty(); } - double[] validCollection = collection - .Where(x => !double.IsNaN(x) && !double.IsInfinity(x)) - .ToArray(); + foreach (double value in collection) + { + if (!double.IsNaN(value) && !double.IsInfinity(value)) + { + validCollection.Add(value); + } + } + if (validCollection.Count == 0) + { + return Array.Empty(); + } // Calculate Q1 (25th percentile) and Q3 (75th percentile) double q1 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.25); double q3 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.75); From 99aea958d7daccc2d1f798d203ba5ed998910074 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 15:23:03 +0800 Subject: [PATCH 40/54] check if TraceConfigurations is null --- .../Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index f684848044a..f5363d063ca 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -120,7 +120,7 @@ public static ConcurrentBag ConcurrentBag microbenchmarkResults = new(); Dictionary jsonToTraceMap = new(); - if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") + if ((!excludeTraces) && configuration.TraceConfigurations?.Type != "none") { jsonToTraceMap = MapJsonToTrace(configuration.Output.Path, bdnJsonResults); } From 2ba985f90244612f538ba0637bb69867d7e47556 Mon Sep 17 00:00:00 2001 From: VincentBu <44959937+VincentBu@users.noreply.github.com> Date: Mon, 18 May 2026 15:32:19 +0800 Subject: [PATCH 41/54] Check if first baseline/comparand is null Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../Microbenchmarks/MicrobenchmarkComparisonResult.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index a2c2e1dbfc4..879d665037a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -46,9 +46,12 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline } } - BaselineRunName = baselines?.FirstOrDefault()?.Parent?.Name; - ComparandRunName = comparands?.FirstOrDefault()?.Parent?.Name; - MicrobenchmarkName = baselines?.FirstOrDefault()?.MicrobenchmarkName; + var firstBaseline = baselines?.FirstOrDefault(); + var firstComparand = comparands?.FirstOrDefault(); + + BaselineRunName = firstBaseline?.Parent?.Name ?? string.Empty; + ComparandRunName = firstComparand?.Parent?.Name ?? string.Empty; + MicrobenchmarkName = firstBaseline?.MicrobenchmarkName ?? string.Empty; Baselines = baselines ?? new List(); Comparands = comparands ?? new List(); From 6008caea4d64c97a01c09b8de7f5a9a3932e123a Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 15:35:33 +0800 Subject: [PATCH 42/54] skip null or empty MicrobenchmarkResult collection --- .../Microbenchmarks/MicrobenchmarkResultComparison.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index f5363d063ca..a993e214186 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -258,6 +258,16 @@ public static List CompareMicrobenchmarkResults( var baselineMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.Name == baselineName); var comparandMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.Name == runName); + if (baselineMicrobenchmarkResults == null || comparandMicrobenchmarkResults ==null) + { + continue; + } + + if (baselineMicrobenchmarkResults.Count == 0 || comparandMicrobenchmarkResults.Count == 0) + { + continue; + } + lock (_lock) { comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); From 205de19a7a69ce7dd4a1b6b33b32c2ca7fbf187d Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 15:36:17 +0800 Subject: [PATCH 43/54] skip null or empty MicrobenchmarkResults collection --- .../Microbenchmarks/MicrobenchmarkResultComparison.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index a993e214186..3d390e03ea2 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -281,6 +281,15 @@ public static List CompareMicrobenchmarkResults( var baselineMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.is_baseline); var comparandMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => !r.Parent.is_baseline); + if (baselineMicrobenchmarkResults == null || comparandMicrobenchmarkResults == null) + { + return; + } + + if (baselineMicrobenchmarkResults.Count == 0 || comparandMicrobenchmarkResults.Count == 0) + { + return; + } lock (_lock) { comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); From 2f140a0d6b4655b4e2c7a7cf7b06b6a3fb6c1d9f Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 15:50:19 +0800 Subject: [PATCH 44/54] apply q1 and q3 on valid collection --- .../gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs index 609066f035a..5f0309b21dd 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs @@ -62,7 +62,7 @@ public static IEnumerable RemoveOutliers(IEnumerable collection) double upperBound = q3 + 1.5 * iqr; // Filter out outliers - return GoodLinq.Where(collection, x => x >= lowerBound && x <= upperBound); + return GoodLinq.Where(validCollection, x => x >= lowerBound && x <= upperBound); } } } From c988e787977e3681b11a1be7233aeccaab45273a Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 15:58:07 +0800 Subject: [PATCH 45/54] use run.Name instead of trace path --- .../Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 3d390e03ea2..fc61dbbfadf 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -195,7 +195,7 @@ public static ConcurrentBag run, benchmark, gcData: benchmarkGCData, - gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, tracePath, benchmark.FullName), + gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, run.Name!, benchmark.FullName), additionalReportMetrics: configuration.Output.additional_report_metrics, cpuColumns: configuration.Output.cpu_columns, columns: configuration.Output.Columns); From 57d227269731681ca4a7a148e91436ba1f26e1ab Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 16:00:53 +0800 Subject: [PATCH 46/54] check possible null value --- .../Microbenchmarks/MicrobenchmarkResultComparison.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index fc61dbbfadf..df7fba94040 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -87,7 +87,7 @@ public static Dictionary MapJsonToTrace(string outputPath, Concu throw new InvalidOperationException($"Benchmark name {benchmarkName} does not have a corresponding trace file pattern in the map."); } var traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkName]; - string outputPathForRun = Path.Combine(outputPath, run.Name); + string outputPathForRun = Path.Combine(outputPath, run.Name!); var sortedTraceFiles = Directory.GetFiles(outputPathForRun, $"{traceFileNameTemplate}*.etl.zip", SearchOption.TopDirectoryOnly) .OrderBy(traceFile => { @@ -146,11 +146,14 @@ public static ConcurrentBag return; } - if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") + if ((!excludeTraces) && configuration.TraceConfigurations?.Type != "none") { string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name!); - string tracePath = jsonToTraceMap.GetValueOrDefault(jsonPath, ""); + if (!jsonToTraceMap.TryGetValue(jsonPath, out string? tracePath) || string.IsNullOrWhiteSpace(tracePath)) + { + throw new InvalidOperationException($"Trace collection is enabled, but no trace path mapping was found for benchmark result '{jsonPath}'."); + } using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) { List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); From 2a89b387ff01a460769b7988e1f8f76ef94a1c09 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 17:00:08 +0800 Subject: [PATCH 47/54] check if metric is in OtherMetricsDiffPerc --- .../Presentation/Microbenchmarks/Markdown.cs | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index 17f03471a5c..c3a04ca54e3 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -1,7 +1,6 @@ using API = GC.Analysis.API; using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Analysis.API; using GC.Infrastructure.Core.Configurations.Microbenchmarks; namespace GC.Infrastructure.Core.Presentation.Microbenchmarks @@ -18,15 +17,15 @@ public static void GenerateTable(MicrobenchmarkConfiguration configuration, IRea // Create summary. sw.WriteLine("# Summary"); - string header = $"| Criteria | {string.Join("|", GoodLinq.Select(comparisonResultsCollection, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; + string header = $"| Criteria | {string.Join("|", API.GoodLinq.Select(comparisonResultsCollection, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; sw.WriteLine(header); sw.WriteLine($"| ----- | {string.Join("|", Enumerable.Repeat(" ----- ", comparisonResultsCollection.Count))} |"); - sw.WriteLine($"| Large Regressions (>20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.LargeRegressions.Count())}|"); - sw.WriteLine($"| Regressions (5% - 20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.Regressions.Count())}|"); - sw.WriteLine($"| Stale Regressions (0% - 5%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.StaleRegressions.Count())}|"); - sw.WriteLine($"| Stale Improvements (0% - 5%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.StaleImprovements.Count())}|"); - sw.WriteLine($"| Improvements (5% - 20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.Improvements.Count())}|"); - sw.WriteLine($"| Large Improvements (>20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.LargeImprovements.Count())}|"); + sw.WriteLine($"| Large Regressions (>20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.LargeRegressions.Count())}|"); + sw.WriteLine($"| Regressions (5% - 20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.Regressions.Count())}|"); + sw.WriteLine($"| Stale Regressions (0% - 5%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.StaleRegressions.Count())}|"); + sw.WriteLine($"| Stale Improvements (0% - 5%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.StaleImprovements.Count())}|"); + sw.WriteLine($"| Improvements (5% - 20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.Improvements.Count())}|"); + sw.WriteLine($"| Large Improvements (>20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.LargeImprovements.Count())}|"); sw.WriteLine($"| Total | {comparisonResultsCollection.Count} |"); sw.WriteLine("\n"); @@ -89,40 +88,42 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben foreach (var metric in configuration.Output.additional_report_metrics) { sw.WriteLine($"## Comparison by {metric}"); - var ordered = comparisonResult.Comparisons.OrderByDescending(c => c.OtherMetricsDiffPerc[metric]); + var ordered = comparisonResult.Comparisons + .Where(c => c.OtherMetricsDiffPerc.ContainsKey(metric)) + .OrderByDescending(c => c.OtherMetricsDiffPerc[metric]); // Large Regressions sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.2)); + sw.AddTableForSingleCriteria(configuration, API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.2)); sw.WriteLine("\n"); // Large Improvements sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); - var largeImprovements = GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.2); + var largeImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.2); largeImprovements.Reverse(); sw.AddTableForSingleCriteria(configuration, largeImprovements); sw.WriteLine("\n"); // Regressions sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); + sw.AddTableForSingleCriteria(configuration, API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); sw.WriteLine("\n"); // Improvements sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); - var improvements = GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.05 && o.OtherMetricsDiffPerc[metric] > -0.2); + var improvements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.05 && o.OtherMetricsDiffPerc[metric] > -0.2); improvements.Reverse(); sw.AddTableForSingleCriteria(configuration, improvements); sw.WriteLine("\n"); // Stale Regressions sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] < 0.05 && o.OtherMetricsDiffPerc[metric] >= 0.0)); + sw.AddTableForSingleCriteria(configuration, API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] < 0.05 && o.OtherMetricsDiffPerc[metric] >= 0.0)); sw.WriteLine("\n"); // Stale Improvements sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); - var staleImprovements = GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -0.05 && o.OtherMetricsDiffPerc[metric] <= 0.0); + var staleImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -0.05 && o.OtherMetricsDiffPerc[metric] <= 0.0); staleImprovements.Reverse(); sw.AddTableForSingleCriteria(configuration, staleImprovements); sw.WriteLine("\n"); From ec4fcc95a23c5bfc5066daed69390495c06c3c4d Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 17:10:32 +0800 Subject: [PATCH 48/54] Add a guard for an empty GC list --- .../MicrobenchmarkResultComparison.cs | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index df7fba94040..a5886a41082 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -194,15 +194,19 @@ public static ConcurrentBag benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); } */ - microbenchmarkResult = new(benchmarkFullName, - run, - benchmark, - gcData: benchmarkGCData, - gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, run.Name!, benchmark.FullName), - additionalReportMetrics: configuration.Output.additional_report_metrics, - cpuColumns: configuration.Output.cpu_columns, - columns: configuration.Output.Columns); - microbenchmarkResults.Add(microbenchmarkResult!); + + if (benchmarkGCData.GCs.Count > 0) + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + gcData: benchmarkGCData, + gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, run.Name!, benchmark.FullName), + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + microbenchmarkResults.Add(microbenchmarkResult!); + } } } } From 09236f7903f88bc6a37af2a8e208d736ff5f8ed0 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 17:39:51 +0800 Subject: [PATCH 49/54] check metric for both baseline and comparand --- .../MicrobenchmarkComparisonResult.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 879d665037a..28639fcc3a7 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -123,8 +123,15 @@ public double MeanDiffPerc{ .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) .ToDictionary(x => x.Item1, x => x.Item2); - public Dictionary OtherMetricsDiff => OutliersFreeBaselineOtherMetrics - .Select(kvp => (kvp.Key, AveragedComparandOtherMetrics[kvp.Key] - AveragedBaselineOtherMetrics[kvp.Key])) + public Dictionary OtherMetricsDiff => AveragedBaselineOtherMetrics + .Select(kvp => + { + if (AveragedComparandOtherMetrics.ContainsKey(kvp.Key)) + { + return (kvp.Key, AveragedComparandOtherMetrics[kvp.Key] - AveragedBaselineOtherMetrics[kvp.Key]); + } + return (kvp.Key, double.NaN); + }) .ToDictionary(x => x.Item1, x => x.Item2); public Dictionary OtherMetricsDiffPerc => OutliersFreeBaselineOtherMetrics @@ -141,6 +148,12 @@ public double MeanDiffPerc{ return (kvp.Key, double.NaN); } } + + if (OtherMetricsDiff[kvp.Key] == double.NaN) + { + return (kvp.Key, double.NaN); + } + return (kvp.Key, OtherMetricsDiff[kvp.Key] / AveragedBaselineOtherMetrics[kvp.Key]); }) .ToDictionary(x => x.Item1, x => x.Item2); From b86a44025a78bc3b4a1d96a9814dabd6ce516e63 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Mon, 18 May 2026 17:39:58 +0800 Subject: [PATCH 50/54] formatting --- .../MicrobenchmarkResultComparison.cs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index a5886a41082..6d652148857 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -198,13 +198,13 @@ public static ConcurrentBag if (benchmarkGCData.GCs.Count > 0) { microbenchmarkResult = new(benchmarkFullName, - run, - benchmark, - gcData: benchmarkGCData, - gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, run.Name!, benchmark.FullName), - additionalReportMetrics: configuration.Output.additional_report_metrics, - cpuColumns: configuration.Output.cpu_columns, - columns: configuration.Output.Columns); + run, + benchmark, + gcData: benchmarkGCData, + gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, run.Name!, benchmark.FullName), + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); microbenchmarkResults.Add(microbenchmarkResult!); } } @@ -220,11 +220,11 @@ public static ConcurrentBag MicrobenchmarkResult? microbenchmarkResult = null; microbenchmarkResult = new(benchmarkFullName, - run, - benchmark, - additionalReportMetrics: configuration.Output.additional_report_metrics, - cpuColumns: configuration.Output.cpu_columns, - columns: configuration.Output.Columns); + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); microbenchmarkResults.Add(microbenchmarkResult!); } } From aa9e68cfb38fead3a37a7a8a95dafd796f1dddb1 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Tue, 19 May 2026 14:11:39 +0800 Subject: [PATCH 51/54] check keys existence --- .../Microbenchmarks/MicrobenchmarkComparisonResult.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 28639fcc3a7..b927683e79b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -134,9 +134,13 @@ public double MeanDiffPerc{ }) .ToDictionary(x => x.Item1, x => x.Item2); - public Dictionary OtherMetricsDiffPerc => OutliersFreeBaselineOtherMetrics + public Dictionary OtherMetricsDiffPerc => AveragedBaselineOtherMetrics .Select(kvp => { + if (!AveragedComparandOtherMetrics.ContainsKey(kvp.Key)) + { + return (kvp.Key, double.NaN); + } if (AveragedBaselineOtherMetrics[kvp.Key] == 0) { if (AveragedComparandOtherMetrics[kvp.Key] == 0) @@ -149,6 +153,11 @@ public double MeanDiffPerc{ } } + if (!OtherMetricsDiff.ContainsKey(kvp.Key)) + { + return (kvp.Key, double.NaN); + } + if (OtherMetricsDiff[kvp.Key] == double.NaN) { return (kvp.Key, double.NaN); From 967d19ba57e10625ffeb209b7b00683ef2131114 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Tue, 19 May 2026 14:22:49 +0800 Subject: [PATCH 52/54] don't drop bdn results even gcprocessdata are not found --- .../MicrobenchmarkResultComparison.cs | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 6d652148857..fad690bb7e4 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -205,9 +205,28 @@ public static ConcurrentBag additionalReportMetrics: configuration.Output.additional_report_metrics, cpuColumns: configuration.Output.cpu_columns, columns: configuration.Output.Columns); - microbenchmarkResults.Add(microbenchmarkResult!); + } + else + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); } } + else + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + + microbenchmarkResults.Add(microbenchmarkResult!); } } } From 9c1aa24c4ce3716edadedde4fce5a2bb2810fa76 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Tue, 19 May 2026 14:26:07 +0800 Subject: [PATCH 53/54] fix possible bugs --- .../Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index fad690bb7e4..9e82ebae66e 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -120,7 +120,7 @@ public static ConcurrentBag ConcurrentBag microbenchmarkResults = new(); Dictionary jsonToTraceMap = new(); - if ((!excludeTraces) && configuration.TraceConfigurations?.Type != "none") + if ((!excludeTraces) && (configuration.TraceConfigurations?.Type ?? "none") != "none") { jsonToTraceMap = MapJsonToTrace(configuration.Output.Path, bdnJsonResults); } From f9245d164ea8fba669269018eb3dd64aaf6146b9 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Tue, 19 May 2026 17:30:34 +0800 Subject: [PATCH 54/54] fix bugs for addtional metrics comparison results presentation --- .../MicrobenchmarkComparisonResult.cs | 2 +- .../Presentation/Microbenchmarks/Markdown.cs | 66 ++++++++++++------- 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index b927683e79b..c2f5c40336b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -163,7 +163,7 @@ public double MeanDiffPerc{ return (kvp.Key, double.NaN); } - return (kvp.Key, OtherMetricsDiff[kvp.Key] / AveragedBaselineOtherMetrics[kvp.Key]); + return (kvp.Key, 100 * OtherMetricsDiff[kvp.Key] / AveragedBaselineOtherMetrics[kvp.Key]); }) .ToDictionary(x => x.Item1, x => x.Item2); } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index c3a04ca54e3..dca122b017a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -1,15 +1,13 @@ -using API = GC.Analysis.API; -using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using System.Configuration; +using API = GC.Analysis.API; namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Markdown { - private const string baseTableString = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; - private const string baseTableRows = "| --- | --- | -- | --- | --- | --- | --- | "; - public static void GenerateTable(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResultsCollection, Dictionary executionDetails, string path) { using (StreamWriter sw = new StreamWriter(path)) @@ -93,49 +91,60 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben .OrderByDescending(c => c.OtherMetricsDiffPerc[metric]); // Large Regressions - sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.2)); + var largeRegression = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 20); + sw.WriteLine($"### Large Regressions (>20%): {largeRegression.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, largeRegression, metric); sw.WriteLine("\n"); // Large Improvements - sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); - var largeImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.2); + var largeImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -20); largeImprovements.Reverse(); - sw.AddTableForSingleCriteria(configuration, largeImprovements); + sw.WriteLine($"### Large Improvements (>20%): {largeImprovements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, largeImprovements, metric); sw.WriteLine("\n"); // Regressions - sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); + var regressions = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 5 && o.OtherMetricsDiffPerc[metric] < 20); + sw.WriteLine($"### Regressions (5% - 20%): {regressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, regressions, metric); sw.WriteLine("\n"); // Improvements - sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); - var improvements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -0.05 && o.OtherMetricsDiffPerc[metric] > -0.2); + var improvements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -5 && o.OtherMetricsDiffPerc[metric] > -20); improvements.Reverse(); - sw.AddTableForSingleCriteria(configuration, improvements); + sw.WriteLine($"### Improvements (5% - 20%): {improvements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, improvements, metric); sw.WriteLine("\n"); // Stale Regressions - sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] < 0.05 && o.OtherMetricsDiffPerc[metric] >= 0.0)); + var staleRegressions = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.0 && o.OtherMetricsDiffPerc[metric] < 5); + sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {staleRegressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, staleRegressions, metric); sw.WriteLine("\n"); // Stale Improvements - sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); - var staleImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -0.05 && o.OtherMetricsDiffPerc[metric] <= 0.0); + var staleImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -5 && o.OtherMetricsDiffPerc[metric] <= 0.0); staleImprovements.Reverse(); - sw.AddTableForSingleCriteria(configuration, staleImprovements); + sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {staleImprovements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, staleImprovements, metric); sw.WriteLine("\n"); } } } - internal static void AddTableForSingleCriteria(this StreamWriter sw, MicrobenchmarkConfiguration configuration, IEnumerable comparisons) + internal static void AddTableForSingleCriteria(this StreamWriter sw, MicrobenchmarkConfiguration configuration, IEnumerable comparisons, string? metricName = null) { // Check if all comparisons have traces. - string tableHeader0 = baseTableString; - string tableHeader1 = baseTableRows; + string tableHeader0 = ""; + if (!string.IsNullOrEmpty(metricName)) + { + tableHeader0 = $"| Benchmark Name | Baseline | Comparand | Baseline {metricName} | Comparand {metricName} | Δ {metricName} | Δ% {metricName} |"; + } + else + { + tableHeader0 = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; + } + string tableHeader1 = "| --- | --- | -- | --- | --- | --- | --- | "; if (configuration.Output.Columns != null) { @@ -164,7 +173,16 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm try { string benchmarkName = lr.MicrobenchmarkName.Replace("<", "\\<").Replace(">", "\\>"); - var baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineMeanValue, 2)} | {Math.Round(lr.AveragedComparandMeanValue, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; + string baseRow = ""; + + if (!String.IsNullOrEmpty(metricName)) + { + baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineOtherMetrics[metricName], 2)} | {Math.Round(lr.AveragedComparandOtherMetrics[metricName], 2)} | {Math.Round(lr.OtherMetricsDiff[metricName], 2)}| {Math.Round(lr.OtherMetricsDiffPerc[metricName], 2)}|"; + } + else + { + baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineMeanValue, 2)} | {Math.Round(lr.AveragedComparandMeanValue, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; + } if (configuration.Output.Columns != null) {