Skip to content

Commit 0144a67

Browse files
SockworkOrangeSockworkOrange
committed
fix: refactor benchmark parameter configurations for readability
1 parent 076be0a commit 0144a67

9 files changed

Lines changed: 147 additions & 79 deletions

‎benchmark/BenchmarkRunner/BenchmarkRunner.csproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
@@ -27,6 +27,7 @@
2727
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
2828
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.2" />
2929
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
30+
<PackageReference Include="NUnit" Version="4.2.2" />
3031
</ItemGroup>
3132

3233
<ItemGroup>

‎benchmark/BenchmarkRunner/Benchmarks/BaseReadBenchmark.cs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
using BenchmarkDotNet.Attributes;
22
using BenchmarkRunner.Utils;
33

4+
public readonly record struct ReadBenchmarkParams(
5+
int Limit,
6+
int Concurrency,
7+
int QueriesToSubmit
8+
);
9+
410
public abstract class BaseReadBenchmark
511
{
6-
[Params(100, 500)]
7-
public int Limit { get; set; }
8-
912
[IterationSetup]
1013
public static void IterationSetup() => Helpers.InvokeGarbageCollection();
1114
public abstract Task Sqlc_GetCustomerOrders();

‎benchmark/BenchmarkRunner/Benchmarks/BaseWriteBenchmark.cs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
using BenchmarkDotNet.Attributes;
1+
public readonly record struct WriteBenchmarkArgs(
2+
int TotalRecords,
3+
int BatchSize
4+
);
25

36
public abstract class BaseWriteBenchmark
47
{
@@ -9,6 +12,6 @@ public abstract class BaseWriteBenchmark
912
ItemsPerOrder: 0
1013
);
1114

12-
public abstract Task Sqlc_AddOrderItems(int batchSize);
13-
public abstract Task EFCore_AddOrderItems(int batchSize);
15+
public abstract Task Sqlc_AddOrderItems(WriteBenchmarkArgs args);
16+
public abstract Task EFCore_AddOrderItems(WriteBenchmarkArgs args);
1417
}

‎benchmark/BenchmarkRunner/Benchmarks/MysqlReadBenchmark.cs‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using BenchmarkRunner.Utils;
55
using MysqlEFCoreImpl;
66
using MysqlSqlcImpl;
7+
using NUnit.Framework;
78

89
namespace BenchmarkRunner.Benchmarks;
910

@@ -17,55 +18,67 @@ public class MysqlReadBenchmark : BaseReadBenchmark
1718
private static readonly string _connectionString = Config.GetMysqlConnectionString();
1819
private readonly QuerySql _sqlcImpl = new(_connectionString);
1920

20-
[Params(3_000)]
21-
public int QueriesToRun { get; set; }
21+
public static IEnumerable<ReadBenchmarkParams> GetParams()
22+
{
23+
yield return new ReadBenchmarkParams(Limit: 50, Concurrency: 200, QueriesToSubmit: 4_000);
24+
yield return new ReadBenchmarkParams(Limit: 1000, Concurrency: 100, QueriesToSubmit: 2_000);
25+
}
2226

23-
[Params(200, 500)]
24-
public int ConcurrentQueries { get; set; }
27+
[ParamsSource(nameof(GetParams))]
28+
public ReadBenchmarkParams Params { get; set; }
2529

2630
[BenchmarkCategory("Read")]
2731
[Benchmark(Baseline = true, Description = "SQLC - GetCustomerOrders")]
2832
public override async Task Sqlc_GetCustomerOrders()
2933
{
30-
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, _ =>
34+
await ExecuteConcurrentlyAsync(Params.QueriesToSubmit, Params.Concurrency, async _ =>
3135
{
32-
return _sqlcImpl.GetCustomerOrdersAsync(new QuerySql.GetCustomerOrdersArgs(
36+
var results = await _sqlcImpl.GetCustomerOrdersAsync(new QuerySql.GetCustomerOrdersArgs(
3337
CustomerId: Random.Shared.Next(1, GetSeedConfig().CustomerCount),
3438
Offset: 0,
35-
Limit: Limit
39+
Limit: Params.Limit
3640
));
41+
42+
Assert.Equals(results.Count, Params.Limit);
43+
return results;
3744
});
3845
}
3946

4047
[BenchmarkCategory("Read")]
4148
[Benchmark(Description = "EFCore (NoTracking) - GetCustomerOrders")]
4249
public override async Task EFCore_NoTracking_GetCustomerOrders()
4350
{
44-
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
51+
await ExecuteConcurrentlyAsync(Params.QueriesToSubmit, Params.Concurrency, async _ =>
4552
{
4653
await using var dbContext = new SalesDbContext(_connectionString);
4754
var queries = new Queries(dbContext, useTracking: false);
48-
return await queries.GetCustomerOrders(new Queries.GetCustomerOrdersArgs(
55+
var results = await queries.GetCustomerOrders(new Queries.GetCustomerOrdersArgs(
4956
CustomerId: Random.Shared.Next(1, GetSeedConfig().CustomerCount),
5057
Offset: 0,
51-
Limit: Limit
58+
Limit: Params.Limit
5259
));
60+
61+
Assert.Equals(results.Count, Params.Limit);
62+
return results;
5363
});
5464
}
5565

5666
[BenchmarkCategory("Read")]
5767
[Benchmark(Description = "EFCore (WithTracking) - GetCustomerOrders")]
5868
public override async Task EFCore_WithTracking_GetCustomerOrders()
5969
{
60-
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
70+
await ExecuteConcurrentlyAsync(Params.QueriesToSubmit, Params.Concurrency, async _ =>
6171
{
6272
await using var dbContext = new SalesDbContext(_connectionString);
6373
var queries = new Queries(dbContext, useTracking: true);
64-
return await queries.GetCustomerOrders(new Queries.GetCustomerOrdersArgs(
74+
var results = await queries.GetCustomerOrders(new Queries.GetCustomerOrdersArgs(
6575
CustomerId: Random.Shared.Next(1, GetSeedConfig().CustomerCount),
6676
Offset: 0,
67-
Limit: Limit
77+
Limit: Params.Limit
6878
));
79+
80+
Assert.Equals(results.Count, Params.Limit);
81+
return results;
6982
});
7083
}
7184

‎benchmark/BenchmarkRunner/Benchmarks/MysqlWriteBenchmark.cs‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,27 +21,34 @@ public class MysqlWriteBenchmark : BaseWriteBenchmark
2121
private readonly Queries _efCoreImpl = new(new SalesDbContext(_connectionString), useTracking: false);
2222
private List<QuerySql.AddOrderItemsArgs> _testOrderItems = null!;
2323

24-
[Params(200_000)]
25-
public int TotalRecords { get; set; }
24+
public static IEnumerable<WriteBenchmarkArgs> GetSqlcArguments()
25+
{
26+
yield return new WriteBenchmarkArgs(TotalRecords: 200_000, BatchSize: 1_000);
27+
yield return new WriteBenchmarkArgs(TotalRecords: 200_000, BatchSize: 5_000);
28+
}
2629

2730
[BenchmarkCategory("Write")]
2831
[Benchmark(Baseline = true, Description = "SQLC - AddOrderItems")]
29-
[Arguments(1_000)]
30-
[Arguments(2_000)]
31-
public override async Task Sqlc_AddOrderItems(int batchSize)
32+
[ArgumentsSource(nameof(GetSqlcArguments))]
33+
public override async Task Sqlc_AddOrderItems(WriteBenchmarkArgs args)
34+
{
35+
await Helpers.InsertInBatchesAsync(_testOrderItems, args.BatchSize, _sqlcImpl.AddOrderItemsAsync);
36+
}
37+
38+
public static IEnumerable<WriteBenchmarkArgs> GetEFCoreArguments()
3239
{
33-
await Helpers.InsertInBatchesAsync(_testOrderItems, batchSize, _sqlcImpl.AddOrderItemsAsync);
40+
yield return new WriteBenchmarkArgs(TotalRecords: 200_000, BatchSize: 500);
3441
}
3542

3643
[BenchmarkCategory("Write")]
3744
[Benchmark(Description = "EFCore - AddOrderItems")]
38-
[Arguments(500)]
39-
public override async Task EFCore_AddOrderItems(int batchSize)
45+
[ArgumentsSource(nameof(GetEFCoreArguments))]
46+
public override async Task EFCore_AddOrderItems(WriteBenchmarkArgs args)
4047
{
41-
var args = _testOrderItems.Select(i => new Queries.AddOrderItemsArgs(
48+
var batchArgs = _testOrderItems.Select(i => new Queries.AddOrderItemsArgs(
4249
i.OrderId, i.ProductId, i.Quantity, i.UnitPrice
4350
)).ToList();
44-
await Helpers.InsertInBatchesAsync(args, batchSize, _efCoreImpl.AddOrderItems);
51+
await Helpers.InsertInBatchesAsync(batchArgs, args.BatchSize, _efCoreImpl.AddOrderItems);
4552
}
4653

4754
public static Func<Task> GetSeedMethod()
@@ -59,7 +66,7 @@ public async Task GlobalSetup()
5966
var orderIds = await _sqlcImpl.GetOrderIdsAsync(new QuerySql.GetOrderIdsArgs(Limit: 1000));
6067
var productIds = await _sqlcImpl.GetProductIdsAsync(new QuerySql.GetProductIdsArgs(Limit: 1000));
6168

62-
_testOrderItems = [.. Enumerable.Range(0, TotalRecords).Select(i => new QuerySql.AddOrderItemsArgs(
69+
_testOrderItems = [.. Enumerable.Range(0, 5000).Select(i => new QuerySql.AddOrderItemsArgs(
6370
OrderId: orderIds[i % orderIds.Count].OrderId,
6471
ProductId: productIds[i % productIds.Count].ProductId,
6572
Quantity: Random.Shared.Next(1, 10),

‎benchmark/BenchmarkRunner/Benchmarks/PostgresqlReadBenchmark.cs‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using BenchmarkDotNet.Configs;
33
using BenchmarkDotNet.Jobs;
44
using BenchmarkRunner.Utils;
5+
using NUnit.Framework;
56
using PostgresEFCoreImpl;
67
using PostgresSqlcImpl;
78

@@ -17,55 +18,67 @@ public class PostgresqlReadBenchmark : BaseReadBenchmark
1718
private static readonly string _connectionString = Config.GetPostgresConnectionString();
1819
private readonly QuerySql _sqlcImpl = new(_connectionString);
1920

20-
[Params(1_000)]
21-
public int QueriesToRun { get; set; }
21+
public static IEnumerable<ReadBenchmarkParams> GetParams()
22+
{
23+
yield return new ReadBenchmarkParams(Limit: 50, Concurrency: 100, QueriesToSubmit: 2_000);
24+
yield return new ReadBenchmarkParams(Limit: 1000, Concurrency: 50, QueriesToSubmit: 1_000);
25+
}
2226

23-
[Params(20, 50)]
24-
public int ConcurrentQueries { get; set; }
27+
[ParamsSource(nameof(GetParams))]
28+
public ReadBenchmarkParams Params { get; set; }
2529

2630
[BenchmarkCategory("Read")]
2731
[Benchmark(Baseline = true, Description = "SQLC - GetCustomerOrders")]
2832
public override async Task Sqlc_GetCustomerOrders()
2933
{
30-
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, _ =>
34+
await ExecuteConcurrentlyAsync(Params.QueriesToSubmit, Params.Concurrency, async _ =>
3135
{
32-
return _sqlcImpl.GetCustomerOrdersAsync(new QuerySql.GetCustomerOrdersArgs(
36+
var results = await _sqlcImpl.GetCustomerOrdersAsync(new QuerySql.GetCustomerOrdersArgs(
3337
CustomerId: Random.Shared.Next(1, GetSeedConfig().CustomerCount),
3438
Offset: 0,
35-
Limit: Limit
39+
Limit: Params.Limit
3640
));
41+
42+
Assert.Equals(results.Count, Params.Limit);
43+
return results;
3744
});
3845
}
3946

4047
[BenchmarkCategory("Read")]
4148
[Benchmark(Description = "EFCore (NoTracking) - GetCustomerOrders")]
4249
public override async Task EFCore_NoTracking_GetCustomerOrders()
4350
{
44-
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
51+
await ExecuteConcurrentlyAsync(Params.QueriesToSubmit, Params.Concurrency, async _ =>
4552
{
4653
await using var dbContext = new SalesDbContext(_connectionString);
4754
var queries = new Queries(dbContext, useTracking: false);
48-
return await queries.GetCustomerOrders(new Queries.GetCustomerOrdersArgs(
55+
var results = await queries.GetCustomerOrders(new Queries.GetCustomerOrdersArgs(
4956
CustomerId: Random.Shared.Next(1, GetSeedConfig().CustomerCount),
5057
Offset: 0,
51-
Limit: Limit
58+
Limit: Params.Limit
5259
));
60+
61+
Assert.Equals(results.Count, Params.Limit);
62+
return results;
5363
});
5464
}
5565

5666
[BenchmarkCategory("Read")]
5767
[Benchmark(Description = "EFCore (WithTracking) - GetCustomerOrders")]
5868
public override async Task EFCore_WithTracking_GetCustomerOrders()
5969
{
60-
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
70+
await ExecuteConcurrentlyAsync(Params.QueriesToSubmit, Params.Concurrency, async _ =>
6171
{
6272
await using var dbContext = new SalesDbContext(_connectionString);
6373
var queries = new Queries(dbContext, useTracking: true);
64-
return await queries.GetCustomerOrders(new Queries.GetCustomerOrdersArgs(
74+
var results = await queries.GetCustomerOrders(new Queries.GetCustomerOrdersArgs(
6575
CustomerId: Random.Shared.Next(1, GetSeedConfig().CustomerCount),
6676
Offset: 0,
67-
Limit: Limit
77+
Limit: Params.Limit
6878
));
79+
80+
Assert.Equals(results.Count, Params.Limit);
81+
return results;
6982
});
7083
}
7184

‎benchmark/BenchmarkRunner/Benchmarks/PostgresqlWriteBenchmark.cs‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,27 +20,34 @@ public class PostgresqlWriteBenchmark : BaseWriteBenchmark
2020
private readonly Queries _efCoreImpl = new(new SalesDbContext(_connectionString), useTracking: false);
2121
private List<QuerySql.AddOrderItemsArgs> _testOrderItems = null!;
2222

23-
[Params(500_000)]
24-
public int TotalRecords { get; set; }
23+
public static IEnumerable<WriteBenchmarkArgs> GetSqlcArguments()
24+
{
25+
yield return new WriteBenchmarkArgs(TotalRecords: 200_000, BatchSize: 1_000);
26+
yield return new WriteBenchmarkArgs(TotalRecords: 200_000, BatchSize: 2_000);
27+
}
2528

2629
[BenchmarkCategory("Write")]
2730
[Benchmark(Baseline = true, Description = "SQLC - AddOrderItems")]
28-
[Arguments(500)]
29-
[Arguments(1_000)]
30-
public override async Task Sqlc_AddOrderItems(int batchSize)
31+
[ArgumentsSource(nameof(GetSqlcArguments))]
32+
public override async Task Sqlc_AddOrderItems(WriteBenchmarkArgs args)
33+
{
34+
await Helpers.InsertInBatchesAsync(_testOrderItems, args.BatchSize, _sqlcImpl.AddOrderItemsAsync);
35+
}
36+
37+
public static IEnumerable<WriteBenchmarkArgs> GetEFCoreArguments()
3138
{
32-
await Helpers.InsertInBatchesAsync(_testOrderItems, batchSize, _sqlcImpl.AddOrderItemsAsync);
39+
yield return new WriteBenchmarkArgs(TotalRecords: 200_000, BatchSize: 500);
3340
}
3441

3542
[BenchmarkCategory("Write")]
3643
[Benchmark(Description = "EFCore - AddOrderItems")]
37-
[Arguments(500)]
38-
public override async Task EFCore_AddOrderItems(int batchSize)
44+
[ArgumentsSource(nameof(GetEFCoreArguments))]
45+
public override async Task EFCore_AddOrderItems(WriteBenchmarkArgs args)
3946
{
40-
var args = _testOrderItems.Select(i => new Queries.AddOrderItemsArgs(
47+
var batchArgs = _testOrderItems.Select(i => new Queries.AddOrderItemsArgs(
4148
i.OrderId, i.ProductId, i.Quantity, i.UnitPrice
4249
)).ToList();
43-
await Helpers.InsertInBatchesAsync(args, batchSize, _efCoreImpl.AddOrderItems);
50+
await Helpers.InsertInBatchesAsync(batchArgs, args.BatchSize, _efCoreImpl.AddOrderItems);
4451
}
4552

4653
public static Func<Task> GetSeedMethod()
@@ -58,7 +65,7 @@ public async Task GlobalSetup()
5865
var orderIds = await _sqlcImpl.GetOrderIdsAsync(new QuerySql.GetOrderIdsArgs(Limit: 1000));
5966
var productIds = await _sqlcImpl.GetProductIdsAsync(new QuerySql.GetProductIdsArgs(Limit: 1000));
6067

61-
_testOrderItems = [.. Enumerable.Range(0, TotalRecords).Select(i => new QuerySql.AddOrderItemsArgs(
68+
_testOrderItems = [.. Enumerable.Range(0, 5000).Select(i => new QuerySql.AddOrderItemsArgs(
6269
OrderId: orderIds[i % orderIds.Count].OrderId,
6370
ProductId: productIds[i % productIds.Count].ProductId,
6471
Quantity: Random.Shared.Next(1, 10),

0 commit comments

Comments
 (0)