Compare Vortex GPU decompression against a cuDF Parquet read

.github/actions/setup-duckdb/action.yml

@@ -0,0 +1,17 @@
+name: "Setup DuckDB"
+description: "Download the DuckDB CLI and put it on PATH"
+inputs:
+  duckdb_version:
+    description: "Version of the DuckDB CLI"
+    default: "1.5.5"
+runs:
+  using: "composite"
+  steps:
+    - name: Download DuckDB
+      shell: bash
+      run: |
+        wget -qO- \
+          "https://github.com/duckdb/duckdb/releases/download/v${{ inputs.duckdb_version }}/duckdb_cli-linux-amd64.zip" \
+          | funzip > duckdb
+        chmod +x duckdb
+        echo "$PWD" >> "$GITHUB_PATH"

.github/workflows/develop-bench.yml

@@ -67,10 +67,7 @@ jobs:
           enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
 
       - name: Install DuckDB
-        run: |
-          wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb
-          chmod +x duckdb
-          echo "$PWD" >> "$GITHUB_PATH"
+        uses: ./.github/actions/setup-duckdb
 
       - uses: ./.github/actions/system-info
 

.github/workflows/pr-bench-gpu-compress.yml

@@ -28,10 +28,32 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          # `compare-benchmark-jsons.py` resolves its baseline by walking `git rev-list HEAD`,
+          # which reaches only the tip of a shallow clone.
+          fetch-depth: 0
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           enable-sccache: "true"
+      - name: Install DuckDB
+        # The Public BI datasets are converted from CSV to Parquet through the DuckDB CLI.
+        # Without it those datasets cannot build their fixture.
+        uses: ./.github/actions/setup-duckdb
+      - name: Install uv
+        uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
+        with:
+          # Only `.venv-cudf` below is used, so the repository-wide sync would build the
+          # `vortex-python` extension — and its Rust — for nothing.
+          sync: false
+      - name: Install cuDF
+        # The GPU Parquet number is a cuDF `read_parquet`. cuDF ships prebuilt manylinux wheels
+        # on NVIDIA's index, so it stays a runtime dependency and never enters the Rust build.
+        run: |
+          uv venv --python 3.12 .venv-cudf
+          uv pip install --python .venv-cudf \
+            --extra-index-url https://pypi.nvidia.com \
+            cudf-cu12 pandas pyarrow
+          echo "$PWD/.venv-cudf/bin" >> "$GITHUB_PATH"
       - uses: ./.github/actions/system-info
       - name: Display NVIDIA GPU details
         run: |
@@ -57,18 +79,18 @@ jobs:
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
           bash scripts/bench-taskset.sh target/release_debug/compress-bench \
-            --gpu-decompress -d table > gpu-compress.txt
-          cat gpu-compress.txt
+            --gpu-decompress -d gh-json -o results.json
       - name: Publish results
         shell: bash
         run: |
-          {
-            echo "# GPU Compression"
-            echo
-            echo '```text'
-            cat gpu-compress.txt
-            echo '```'
-          } > comment.md
+          set -Eeu -o pipefail -x
+
+          python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz \
+            data.json.gz --no-sign-request
+          gzip -d -c data.json.gz > base.json
+
+          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json \
+            "GPU Compression" > comment.md
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
       - name: Comment PR
         if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false

.github/workflows/pr-bench-runner.yml

@@ -49,10 +49,7 @@ jobs:
           enable-sccache: ${{ github.event.pull_request.head.repo.fork == false && 'true' || 'false' }}
 
       - name: Install DuckDB
-        run: |
-          wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb
-          chmod +x duckdb
-          echo "$PWD" >> "$GITHUB_PATH"
+        uses: ./.github/actions/setup-duckdb
 
       - uses: ./.github/actions/system-info
 

.github/workflows/sql-bench-matrix.yml

@@ -133,10 +133,7 @@ jobs:
           sync: false
 
       - name: Install DuckDB
-        run: |
-          wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb
-          chmod +x duckdb
-          echo "$PWD" >> "$GITHUB_PATH"
+        uses: ./.github/actions/setup-duckdb
 
       - uses: ./.github/actions/system-info
 

benchmarks/compress-bench/Cargo.toml

@@ -27,6 +27,8 @@ itertools = { workspace = true }
 lance-bench = { path = "../lance-bench", optional = true }
 parquet = { workspace = true }
 regex = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
 tempfile = { workspace = true, optional = true }
 tokio = { workspace = true, features = ["full"] }
 tracing = { workspace = true }
@@ -45,7 +47,6 @@ name = "compress-bench"
 test = false
 
 [lib]
-test = false
 
 [lints]
 workspace = true

benchmarks/compress-bench/README.md

@@ -15,13 +15,109 @@ See [`src/main.rs`](./src/main.rs) for the dataset list and CLI flags (`--format
 cargo run -p compress-bench --profile release_debug
 ```
 
-GPU decompression is opt-in and runs only the existing benchmark names allow-listed in
-`src/main.rs`:
+## GPU decompression
+
+`--gpu-decompress` is opt-in, requires the `cuda` feature, and restricts the suite to the
+GPU dataset list in `src/main.rs`. It measures decompression only, for two backends:
+
+- **Vortex** — the file is written with CUDA-compatible BtrBlocks encodings only
+  (`only_cuda_compatible`) and a CUDA flat layout, then decoded on the device all the way to
+  canonical arrays.
+- **Parquet** — the file is rewritten with GPU-friendly writer settings (see below) and read
+  back with [cuDF](https://github.com/rapidsai/cudf)'s `read_parquet`, which performs the
+  whole read on the device: page header decode, codec decompression, dictionary/RLE/plain
+  decoding and column assembly.
+
+Both sides therefore decode all the way to device-resident arrays, which is what makes the
+`vortex:parquet-<codec> gpu ratio decompress time` metric a like-for-like comparison.
 
 ```bash
 cargo run -p compress-bench --profile release_debug \
   --features cuda,unstable_encodings -- --gpu-decompress
+
+# pick the Parquet page codec the GPU file is written with (default: snappy)
+cargo run -p compress-bench --profile release_debug \
+  --features cuda,unstable_encodings -- --gpu-decompress --gpu-parquet-codec zstd
 ```
 
-On Linux, GPU files are read with direct IO (`O_DIRECT`) so repeated iterations measure
-storage bandwidth rather than page-cache hits.
+### cuDF
+
+cuDF has no Rust binding, so the benchmark drives it out of process: it spawns `python3` running
+`scripts/cudf-parquet-read.py`, which imports cuDF from the prebuilt `cudf-cu12` Python package,
+reads the file, and prints its timings back as JSON on stdout. Nothing links against libcudf, so
+cuDF is a runtime requirement rather than a Rust build dependency:
+
+```bash
+uv pip install --extra-index-url https://pypi.nvidia.com cudf-cu12 pandas pyarrow
+```
+
+The clock lives inside that script rather than around the subprocess, so process spawn,
+interpreter start, `import cudf` and CUDA context creation are all excluded; a warm-up read runs
+first for the same reason. The script performs several timed reads per invocation and reports the
+fastest, and the harness then takes its own minimum across `--iterations`.
+
+Both backends read a warm file by default. cuDF runs an untimed warm-up read before the timed
+one, so its timed read hits the page cache; the Vortex reader therefore does **not** use direct
+I/O by default, because `O_DIRECT` would bypass the page cache and compare a Vortex read of the
+disk against a cuDF read of RAM. `--gpu-direct-io` turns it back on to measure storage bandwidth
+instead — a different question, and the resulting ratio is not a decode comparison.
+
+The remaining asymmetry is the transfer path: the Vortex reader uses pinned buffers, while cuDF
+does its own host read and host-to-device copy.
+
+### GPU-friendly Parquet writer settings
+
+Set in `src/gpu/writer.rs`:
+
+| Setting | Value | Why |
+| --- | --- | --- |
+| writer version | `PARQUET_1_0` | v1 pages compress the whole page body; v2 pages put uncompressed levels ahead of the compressed values in the same body. |
+| compression | Snappy (default) or Zstd | Snappy is the Parquet default and has the higher device-side throughput. |
+| dictionary | enabled | Keeps the decompressed payload small; the encoding GPU Parquet readers decode fastest. |
+| data page size | 1 MiB | Large enough to amortize per-page setup, small enough to keep every SM fed. Matches the page size cuDF targets. |
+| data page row limit | 1,000,000 | The 20k-row default caps narrow columns' pages far below 1 MiB. |
+| statistics | chunk-level | Page statistics only inflate the headers a reader has to walk. |
+| row group size | 1,048,576 rows | Shared with the Vortex side as `GPU_ROW_GROUP_SIZE` — see below. |
+
+### Matching physical partitions
+
+A Parquet row group and a Vortex chunk are the same thing for this comparison: the unit the
+reader plans and dispatches over. Both formats are pinned to `GPU_ROW_GROUP_SIZE`
+(1,048,576 rows, Parquet's `DEFAULT_MAX_ROW_GROUP_ROW_COUNT`).
+
+Without this the two are not comparable. Parquet reads ~1M-row row groups, while the Vortex
+side inherits the Arrow reader's ~8K-row batches — each of which becomes its own chunk, its own
+compressed blocks and its own kernel launches, so a single dispatch turns into hundreds.
+
+Setting the Arrow reader's batch size alone is not enough: the reader also breaks at the source
+file's row group boundaries, so short batches survive. `parquet_to_vortex_chunks_with_batch_size`
+therefore concatenates the source batches and re-slices them on exact boundaries. Those batches
+are written straight through as root chunks via `ChunkedLayoutStrategy`, and read back with
+`SplitBy::RowCount(GPU_ROW_GROUP_SIZE)` so a scan batch is one whole partition.
+
+### Correctness
+
+`--gpu-verify` cross-checks device output against the CPU decoders on every iteration:
+
+- Parquet: the cuDF-read frame is compared against a CPU Parquet read of the same file.
+- Vortex: each GPU-decoded field is copied back and compared against the same field decoded
+  on the CPU, through Arrow with a pinned target type.
+
+The check runs before each timed measurement and is not included in it, so a verifying run
+still publishes comparable numbers — it just takes considerably longer:
+
+```bash
+cargo run -p compress-bench --profile release_debug \
+  --features cuda,unstable_encodings -- --gpu-decompress --gpu-verify --iterations 1
+```
+
+Any `--gpu-decompress` run reports on every dataset rather than stopping at the first failure, so
+one run shows which datasets decode correctly on the GPU and which do not. The timing tables are
+rendered before the failure summary, so a dataset the GPU cannot decode still leaves the rest of
+the matrix with numbers — the process exits non-zero either way.
+
+The dataset list in `src/main.rs` therefore holds only datasets a `--gpu-verify` run has confirmed.
+Several others are waiting on `vortex-cuda` kernel gaps (`u16` in `date_time_parts`, a
+`vortex.masked` kernel, and a CPU fallback reached with device-resident buffers); they are listed
+with their reasons next to `gpu_datasets`. Add one there once its gap is closed and verification
+passes.

benchmarks/compress-bench/src/gpu/mod.rs

@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! The GPU decompression mode behind `--gpu-decompress`.
+//!
+//! This module is the only place in the crate that mentions the `cuda` feature. The two device
+//! backends need it, so they are gated here and reached through [`compressor`]; the rest of the
+//! crate calls that one function and stays feature-agnostic.
+//!
+//! [`GpuOptions`] and [`writer`] deliberately sit outside the gate. Neither touches CUDA, and
+//! keeping them unconditional is what lets `main` parse the `--gpu-*` flags — and reject them
+//! with a clear message — in a build without the feature.
+
+use vortex_bench::Format;
+use vortex_bench::compress::Compressor;
+
+pub mod writer;
+
+#[cfg(feature = "cuda")]
+mod parquet;
+#[cfg(feature = "cuda")]
+mod vortex;
+
+pub use crate::gpu::writer::GpuCodec;
+
+/// Settings for the GPU decompression mode.
+#[derive(Clone, Copy, Debug)]
+pub struct GpuOptions {
+    /// Parquet page codec to write the GPU file with.
+    pub codec: GpuCodec,
+    /// Cross-check decompressed output against the CPU decoders.
+    pub verify: bool,
+    /// Read the Vortex file with direct IO instead of through the page cache.
+    pub direct_io: bool,
+}
+
+/// The GPU backend that measures `format`.
+#[cfg(feature = "cuda")]
+pub fn compressor(format: Format, options: GpuOptions) -> Box<dyn Compressor> {
+    match format {
+        Format::OnDiskVortex => Box::new(vortex::GpuVortexCompressor::new(
+            options.verify,
+            options.direct_io,
+        )) as Box<dyn Compressor>,
+        Format::Parquet => Box::new(parquet::GpuParquetCompressor::new(
+            options.codec,
+            options.verify,
+        )),
+        _ => unimplemented!("GPU compress bench not implemented for {format}"),
+    }
+}
+
+/// Stands in for [`compressor`] in a build without the `cuda` feature.
+///
+/// `main` rejects `--gpu-decompress` before any compressor is selected, so reaching this is a
+/// bug. Destructuring the options is what marks their fields as read: they are only otherwise
+/// used by the gated backends, and without this they are dead code in a non-CUDA build.
+#[cfg(not(feature = "cuda"))]
+pub fn compressor(format: Format, options: GpuOptions) -> Box<dyn Compressor> {
+    let GpuOptions {
+        codec: _,
+        verify: _,
+        direct_io: _,
+    } = options;
+    unreachable!("GPU mode requires the cuda feature, checked before selecting a {format} backend")
+}