From 5112aaf34faef459b8b98f01bf45b30f06ccdf62 Mon Sep 17 00:00:00 2001 From: HarshilMital Date: Sat, 24 Oct 2026 10:27:36 +0000 Subject: [PATCH 1/2] bench: add a compressed high-partition writer case (#7080) Part of #7080. What's added: the shuffle_writer_high_partition group (200, 2000 and 8000 hash partitions, end to end) now runs each partition count with Lz4Frame, the default codec, next to the existing CompressionCodec::None rows. The existing IDs are unchanged. Code path: the per-partition write path with compression on. At 8000 partitions each partition gets about ten rows, so every block is tiny and still pays for a fresh LZ4 frame encoder, its header and end mark; the None row at the same partition count isolates that per-block cost. sample_size(10), as the group already uses. Reproduce on the VM (from native/): cargo bench -p datafusion-comet-shuffle --bench shuffle_writer -- 'shuffle_writer_high_partition/.*compression=Lz4Frame' --save-baseline main Results: TODO (VM) --- native/shuffle/benches/shuffle_writer.rs | 41 +++++++++++++++--------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/native/shuffle/benches/shuffle_writer.rs b/native/shuffle/benches/shuffle_writer.rs index c099fc0413f..7a2003014c0 100644 --- a/native/shuffle/benches/shuffle_writer.rs +++ b/native/shuffle/benches/shuffle_writer.rs @@ -165,25 +165,34 @@ fn criterion_benchmark(c: &mut Criterion) { // buffered writer per partition), which low counts barely exercise; compression // is disabled to isolate it. Few samples: each iteration is a full end-to-end // write across thousands of partitions. + // + // The Lz4Frame variant is what a wide shuffle runs by default. At 8000 partitions each + // partition gets about ten rows, so every block is tiny and still pays for a fresh LZ4 + // frame encoder, its header and its end mark; comparing it with the None row at the same + // partition count shows how much of the write that per-block cost is. let mut high_partition_group = c.benchmark_group("shuffle_writer_high_partition"); high_partition_group.sample_size(10); for num_partitions in [200usize, 2000, 8000] { - high_partition_group.bench_function( - format!("shuffle_writer: end to end (partitions={num_partitions}, compression=None)"), - |b| { - bench_end_to_end(b, || { - create_shuffle_writer_exec( - CompressionCodec::None, - CometPartitioning::Hash( - vec![Arc::new(Column::new("a", 0))], - num_partitions, - ), - 8192, - 10, - ) - }) - }, - ); + for compression_codec in [CompressionCodec::None, CompressionCodec::Lz4Frame] { + high_partition_group.bench_function( + format!( + "shuffle_writer: end to end (partitions={num_partitions}, compression={compression_codec:?})" + ), + |b| { + bench_end_to_end(b, || { + create_shuffle_writer_exec( + compression_codec.clone(), + CometPartitioning::Hash( + vec![Arc::new(Column::new("a", 0))], + num_partitions, + ), + 8192, + 10, + ) + }) + }, + ); + } } high_partition_group.finish(); } From 74a1109c1aacdfd54e32960df312e0dd990f8567 Mon Sep 17 00:00:00 2001 From: HarshilMital Date: Sun, 25 Oct 2026 11:55:24 +0000 Subject: [PATCH 2/2] bench: describe what the None and Lz4Frame high-partition rows each measure The comment above the high-partition group still said compression is disabled to isolate the per-partition write path, which no longer holds now that the group also runs Lz4Frame. Reword it: the None rows isolate that path, and the Lz4Frame rows add the per-block codec cost on top. --- native/shuffle/benches/shuffle_writer.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/native/shuffle/benches/shuffle_writer.rs b/native/shuffle/benches/shuffle_writer.rs index 7a2003014c0..8a4f27cff72 100644 --- a/native/shuffle/benches/shuffle_writer.rs +++ b/native/shuffle/benches/shuffle_writer.rs @@ -162,14 +162,13 @@ fn criterion_benchmark(c: &mut Criterion) { group.finish(); // High partition counts stress the per-partition write path (one short-lived - // buffered writer per partition), which low counts barely exercise; compression - // is disabled to isolate it. Few samples: each iteration is a full end-to-end - // write across thousands of partitions. - // - // The Lz4Frame variant is what a wide shuffle runs by default. At 8000 partitions each - // partition gets about ten rows, so every block is tiny and still pays for a fresh LZ4 - // frame encoder, its header and its end mark; comparing it with the None row at the same - // partition count shows how much of the write that per-block cost is. + // buffered writer per partition), which low counts barely exercise. The None rows + // isolate that path. The Lz4Frame rows, what a wide shuffle runs by default, add the + // per-block codec cost on top: at 8000 partitions each partition gets about ten rows, + // so every block is tiny and still pays for a fresh LZ4 frame encoder, its header and + // its end mark. Comparing the two rows at the same partition count shows that cost; + // at low counts the smaller compressed output can outweigh it. Few samples: each + // iteration is a full end-to-end write across thousands of partitions. let mut high_partition_group = c.benchmark_group("shuffle_writer_high_partition"); high_partition_group.sample_size(10); for num_partitions in [200usize, 2000, 8000] {