From 0da71d8f385ea916de870c075eb4bf5db5b93549 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Mon, 22 Jun 2026 15:56:17 +0000 Subject: [PATCH 1/3] Fix client-v2 RowBinary: write Nullable marker for nested Tuple/Map elements A Nullable element nested inside a Tuple or Map was serialized without its leading null-marker byte for present (non-null) values, corrupting the RowBinary stream so the server (and the client's own reader) mis-parsed the following bytes. Top-level columns get the marker from writeValuePreamble and serializeArrayData already wrote per-element markers, but serializeTupleData and serializeMapData called serializeData directly and skipped it. Add a serializeNestedData helper that writes the 0x00 (present) / 0x01 (null) marker for nested Nullable elements, and route Tuple elements and Map values through it. The array and top-level paths are unchanged. Fixes: https://github.com/ClickHouse/clickhouse-java/issues/2721 --- .../internal/SerializerUtils.java | 25 ++++++- .../internal/SerializerUtilsTest.java | 70 +++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index cc7e91792..1d1afd219 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -111,6 +111,25 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo } } + /** + * Serializes a value that is nested inside a container ({@code Tuple} or {@code Map}). A nested + * {@code Nullable} element is prefixed with its null-marker byte ({@code 0x00} when present, + * {@code 0x01} when null), as the server expects for {@code Nullable} sub-columns in + * {@code RowBinary}. For a top-level column this marker is instead written by + * {@link com.clickhouse.client.api.data_formats.RowBinaryFormatSerializer#writeValuePreamble}, + * so this helper must only be used for nested elements. + */ + private static void serializeNestedData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException { + if (column.isNullable()) { + if (value == null) { + writeNull(stream); + return; + } + writeNonNull(stream); + } + serializeData(stream, value, column); + } + private static final Map, ClickHouseColumn> PREDEFINED_TYPE_COLUMNS = getPredefinedTypeColumnsMap(); private static Map, ClickHouseColumn> getPredefinedTypeColumnsMap() { @@ -460,12 +479,12 @@ public static void serializeTupleData(OutputStream stream, Object value, ClickHo if (value instanceof List) { List values = (List) value; for (int i = 0; i < values.size(); i++) { - serializeData(stream, values.get(i), column.getNestedColumns().get(i)); + serializeNestedData(stream, values.get(i), column.getNestedColumns().get(i)); } } else if (value.getClass().isArray()) { // TODO: this code uses reflection - we might need to measure it and find faster solution. for (int i = 0; i < Array.getLength(value); i++) { - serializeData(stream, Array.get(value, i), column.getNestedColumns().get(i)); + serializeNestedData(stream, Array.get(value, i), column.getNestedColumns().get(i)); } } else { throw new IllegalArgumentException("Cannot serialize " + value + " as a tuple"); @@ -483,7 +502,7 @@ public static void serializeMapData(OutputStream stream, Object value, ClickHous map.forEach((key, val) -> { try { serializePrimitiveData(stream, key, Objects.requireNonNull(column.getKeyInfo())); - serializeData(stream, val, Objects.requireNonNull(column.getValueInfo())); + serializeNestedData(stream, val, Objects.requireNonNull(column.getValueInfo())); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java index 238682f18..71854e795 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java @@ -9,7 +9,9 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.TimeZone; @Test(groups = {"unit"}) @@ -134,6 +136,74 @@ public void testGeometrySerializationRejectsMalformedList() { ClickHouseColumn.of("v", "Geometry"))); } + @Test + public void testTupleWithNullableElementsRoundTrip() throws Exception { + // Regression for #2721: a Nullable element nested inside a Tuple must be prefixed with its + // null-marker byte (0x00 present, 0x01 null). Before the fix the present-value marker was + // missing, so the reader mis-parsed the bytes that followed. The non-nullable sibling element + // must keep being written without a marker. + ClickHouseColumn column = ClickHouseColumn.of("v", "Tuple(Nullable(String), String)"); + + ByteArrayOutputStream present = new ByteArrayOutputStream(); + SerializerUtils.serializeData(present, Arrays.asList("optional-value", "value-2"), column); + Assert.assertEquals((Object[]) newReader(present.toByteArray()).readValue(column), + new Object[] {"optional-value", "value-2"}); + + ByteArrayOutputStream absent = new ByteArrayOutputStream(); + SerializerUtils.serializeData(absent, Arrays.asList(null, "value-2"), column); + Assert.assertEquals((Object[]) newReader(absent.toByteArray()).readValue(column), + new Object[] {null, "value-2"}); + } + + @Test + public void testMapWithNullableValuesRoundTrip() throws Exception { + // Regression for #2721: a Nullable Map value must be prefixed with its null-marker byte. + ClickHouseColumn column = ClickHouseColumn.of("v", "Map(String, Nullable(String))"); + Map value = new LinkedHashMap<>(); + value.put("k1", "v1"); + value.put("k2", null); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + SerializerUtils.serializeData(out, value, column); + + Map result = newReader(out.toByteArray()).readValue(column); + Assert.assertEquals(result, value); + } + + @Test + public void testTupleWithNullableFixedWidthElementsRoundTrip() throws Exception { + // #2721 is about the marker byte, not the element type: a fixed-width Nullable element + // (here Int32) must also get its marker so the following value bytes are not misaligned. + // Also covers an all-null tuple. + ClickHouseColumn column = ClickHouseColumn.of("v", "Tuple(Nullable(Int32), Nullable(String))"); + + ByteArrayOutputStream present = new ByteArrayOutputStream(); + SerializerUtils.serializeData(present, Arrays.asList(42, "x"), column); + Assert.assertEquals((Object[]) newReader(present.toByteArray()).readValue(column), + new Object[] {42, "x"}); + + ByteArrayOutputStream allNull = new ByteArrayOutputStream(); + SerializerUtils.serializeData(allNull, Arrays.asList(null, null), column); + Assert.assertEquals((Object[]) newReader(allNull.toByteArray()).readValue(column), + new Object[] {null, null}); + } + + @Test + public void testNestedContainerWithNullableRoundTrip() throws Exception { + // #2721: the marker handling must compose through nested containers — here a Map carrying a + // Nullable value sits inside a Tuple, exercising serializeTupleData -> serializeMapData. + ClickHouseColumn column = ClickHouseColumn.of("v", "Tuple(String, Map(String, Nullable(String)))"); + Map inner = new LinkedHashMap<>(); + inner.put("k1", "v1"); + inner.put("k2", null); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + SerializerUtils.serializeData(out, Arrays.asList("id", inner), column); + + Assert.assertEquals((Object[]) newReader(out.toByteArray()).readValue(column), + new Object[] {"id", inner}); + } + private void assertCustomGeoTypeTag(String typeName) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); SerializerUtils.writeDynamicTypeTag(out, ClickHouseColumn.of("v", typeName)); From 4858a6f5a369b78e1752b208cb383ba8e91b1661 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:19:00 +0000 Subject: [PATCH 2/3] Restructure nested-Nullable tests into one data-provider test (review feedback) Address review on #2886: squash the four nested-Nullable round-trip tests into a single TestNG @DataProvider-driven test, drop the issue-referencing/verbose test comments, and broaden coverage to all nested datatypes. The provider now exercises a present Nullable element of every byte-width class (String, FixedString, Int/UInt 8/16/32/64, Float32/64, Bool, UUID, Date, Decimal64, IPv4) nested in a Tuple, the same on the Map value path, both serializeTupleData branches (List and array input), null markers, container compositions including Array(Tuple(Nullable)) (how Nested is encoded), and non-nullable contrast cases that must keep serializing without a marker. A small normalize() helper compares round-tripped values structurally across the Tuple/Array/Map container representations the reader returns. --- .../internal/SerializerUtilsTest.java | 163 +++++++++++------- 1 file changed, 101 insertions(+), 62 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java index 71854e795..e53d61166 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java @@ -4,15 +4,22 @@ import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.value.ClickHouseGeoPolygonValue; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.InetAddress; +import java.time.LocalDate; +import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; +import java.util.UUID; @Test(groups = {"unit"}) public class SerializerUtilsTest { @@ -136,72 +143,104 @@ public void testGeometrySerializationRejectsMalformedList() { ClickHouseColumn.of("v", "Geometry"))); } - @Test - public void testTupleWithNullableElementsRoundTrip() throws Exception { - // Regression for #2721: a Nullable element nested inside a Tuple must be prefixed with its - // null-marker byte (0x00 present, 0x01 null). Before the fix the present-value marker was - // missing, so the reader mis-parsed the bytes that followed. The non-nullable sibling element - // must keep being written without a marker. - ClickHouseColumn column = ClickHouseColumn.of("v", "Tuple(Nullable(String), String)"); - - ByteArrayOutputStream present = new ByteArrayOutputStream(); - SerializerUtils.serializeData(present, Arrays.asList("optional-value", "value-2"), column); - Assert.assertEquals((Object[]) newReader(present.toByteArray()).readValue(column), - new Object[] {"optional-value", "value-2"}); - - ByteArrayOutputStream absent = new ByteArrayOutputStream(); - SerializerUtils.serializeData(absent, Arrays.asList(null, "value-2"), column); - Assert.assertEquals((Object[]) newReader(absent.toByteArray()).readValue(column), - new Object[] {null, "value-2"}); - } - - @Test - public void testMapWithNullableValuesRoundTrip() throws Exception { - // Regression for #2721: a Nullable Map value must be prefixed with its null-marker byte. - ClickHouseColumn column = ClickHouseColumn.of("v", "Map(String, Nullable(String))"); - Map value = new LinkedHashMap<>(); - value.put("k1", "v1"); - value.put("k2", null); + @Test(dataProvider = "nestedNullableData") + public void testNestedNullableRoundTrip(String typeName, Object value) throws Exception { + ClickHouseColumn column = ClickHouseColumn.of("v", typeName); ByteArrayOutputStream out = new ByteArrayOutputStream(); SerializerUtils.serializeData(out, value, column); - Map result = newReader(out.toByteArray()).readValue(column); - Assert.assertEquals(result, value); - } - - @Test - public void testTupleWithNullableFixedWidthElementsRoundTrip() throws Exception { - // #2721 is about the marker byte, not the element type: a fixed-width Nullable element - // (here Int32) must also get its marker so the following value bytes are not misaligned. - // Also covers an all-null tuple. - ClickHouseColumn column = ClickHouseColumn.of("v", "Tuple(Nullable(Int32), Nullable(String))"); - - ByteArrayOutputStream present = new ByteArrayOutputStream(); - SerializerUtils.serializeData(present, Arrays.asList(42, "x"), column); - Assert.assertEquals((Object[]) newReader(present.toByteArray()).readValue(column), - new Object[] {42, "x"}); - - ByteArrayOutputStream allNull = new ByteArrayOutputStream(); - SerializerUtils.serializeData(allNull, Arrays.asList(null, null), column); - Assert.assertEquals((Object[]) newReader(allNull.toByteArray()).readValue(column), - new Object[] {null, null}); - } - - @Test - public void testNestedContainerWithNullableRoundTrip() throws Exception { - // #2721: the marker handling must compose through nested containers — here a Map carrying a - // Nullable value sits inside a Tuple, exercising serializeTupleData -> serializeMapData. - ClickHouseColumn column = ClickHouseColumn.of("v", "Tuple(String, Map(String, Nullable(String)))"); - Map inner = new LinkedHashMap<>(); - inner.put("k1", "v1"); - inner.put("k2", null); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - SerializerUtils.serializeData(out, Arrays.asList("id", inner), column); - - Assert.assertEquals((Object[]) newReader(out.toByteArray()).readValue(column), - new Object[] {"id", inner}); + Object actual = newReader(out.toByteArray()).readValue(column); + Assert.assertEquals(normalize(actual), normalize(value)); + } + + @DataProvider(name = "nestedNullableData") + private Object[][] nestedNullableData() throws Exception { + UUID uuid = UUID.fromString("61f0c404-5cb3-11e7-907b-a6006ad3dba0"); + InetAddress ipv4 = InetAddress.getByName("1.2.3.4"); + return new Object[][] { + // A present Nullable element of each datatype nested in a Tuple, with a trailing + // non-nullable sibling whose bytes misalign if the present-marker is dropped. + {"Tuple(Nullable(String), String)", Arrays.asList("opt", "tail")}, + {"Tuple(Nullable(FixedString(3)), String)", Arrays.asList("abc", "tail")}, + {"Tuple(Nullable(Int8), String)", Arrays.asList((byte) -5, "tail")}, + {"Tuple(Nullable(UInt8), String)", Arrays.asList((short) 200, "tail")}, + {"Tuple(Nullable(Int16), String)", Arrays.asList((short) -1600, "tail")}, + {"Tuple(Nullable(UInt16), String)", Arrays.asList(40000, "tail")}, + {"Tuple(Nullable(Int32), String)", Arrays.asList(42, "tail")}, + {"Tuple(Nullable(UInt32), String)", Arrays.asList(4_000_000_000L, "tail")}, + {"Tuple(Nullable(Int64), String)", Arrays.asList(-64L, "tail")}, + {"Tuple(Nullable(UInt64), String)", Arrays.asList(BigInteger.valueOf(64), "tail")}, + {"Tuple(Nullable(Float32), String)", Arrays.asList(1.5f, "tail")}, + {"Tuple(Nullable(Float64), String)", Arrays.asList(2.5d, "tail")}, + {"Tuple(Nullable(Bool), String)", Arrays.asList(true, "tail")}, + {"Tuple(Nullable(UUID), String)", Arrays.asList(uuid, "tail")}, + {"Tuple(Nullable(Date), String)", Arrays.asList(LocalDate.of(2021, 2, 3), "tail")}, + {"Tuple(Nullable(Decimal64(4)), String)", Arrays.asList(new BigDecimal("1.2345"), "tail")}, + {"Tuple(Nullable(IPv4), String)", Arrays.asList(ipv4, "tail")}, + + // A Tuple value given as a Java array (not a List) takes the other branch of + // serializeTupleData, which is routed through the same nested-marker path. + {"Tuple(Nullable(String), String)", new Object[] {"opt", "tail"}}, + + // The same marker handling on the Map value path, across a range of widths. + {"Map(String, Nullable(String))", newMap("k", "v")}, + {"Map(String, Nullable(Int32))", newMap("k", 32)}, + {"Map(String, Nullable(Float64))", newMap("k", 2.5d)}, + {"Map(String, Nullable(UUID))", newMap("k", uuid)}, + + // Null elements/values still serialize a single null-marker byte. + {"Tuple(Nullable(String), String)", Arrays.asList(null, "tail")}, + {"Tuple(Nullable(Int32), String)", Arrays.asList(null, "tail")}, + {"Tuple(Nullable(Int32), Nullable(String))", Arrays.asList(null, null)}, + {"Map(String, Nullable(String))", newMap("k", null)}, + + // Containers compose: marker handling threads through nested Tuple/Map/Array, + // including Array(Tuple(Nullable)) which is how Nested columns are encoded. + {"Tuple(String, Map(String, Nullable(String)))", Arrays.asList("id", newMap("k1", "v1", "k2", null))}, + {"Tuple(Nullable(String), Map(String, Nullable(Int32)))", Arrays.asList("opt", newMap("k", 7))}, + {"Array(Tuple(Nullable(String), String))", Arrays.asList(Arrays.asList("a", "b"), Arrays.asList(null, "c"))}, + {"Tuple(Array(Nullable(Int32)), String)", Arrays.asList(Arrays.asList(1, null, 3), "tail")}, + + // Contrast: non-nullable nested elements must keep serializing without a marker. + {"Tuple(Int32, String)", Arrays.asList(7, "tail")}, + {"Map(String, String)", newMap("k", "v")}, + }; + } + + // Normalizes Tuple (Object[]) and Array (ArrayValue / List) results to nested Lists so + // round-tripped values compare structurally regardless of the container representation the + // reader returns. + @SuppressWarnings("unchecked") + private static Object normalize(Object value) { + if (value instanceof BinaryStreamReader.ArrayValue) { + return normalizeList(((BinaryStreamReader.ArrayValue) value).asList()); + } else if (value instanceof Object[]) { + return normalizeList(Arrays.asList((Object[]) value)); + } else if (value instanceof List) { + return normalizeList((List) value); + } else if (value instanceof Map) { + Map result = new LinkedHashMap<>(); + ((Map) value).forEach((k, v) -> result.put(k, normalize(v))); + return result; + } + return value; + } + + private static List normalizeList(List values) { + List result = new ArrayList<>(values.size()); + for (Object v : values) { + result.add(normalize(v)); + } + return result; + } + + private static Map newMap(Object... kv) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + map.put(kv[i], kv[i + 1]); + } + return map; } private void assertCustomGeoTypeTag(String typeName) throws Exception { From 3d6a219746d98eec56786ed38347c78eb1fbe9c5 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Mon, 22 Jun 2026 21:47:02 +0000 Subject: [PATCH 3/3] Strengthen nested-Nullable test: place Nullable mid-schema with trailing Float64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback: a single-column / leading-Nullable schema cannot detect a byte-misaligned nested-Nullable serialization, because nothing follows the dropped marker. Each present-Nullable data-provider row now puts the Nullable in the MIDDLE of the schema (Tuple(Int32, Nullable(X), Float64)) with a leading non-null column and a trailing fixed-width Float64, and the assertion compares the whole row — so a dropped not-null marker shifts the following bytes and the trailing Float64 reads a wrong value (or hits EOF), positionally detecting the fault. Also covers the Java-array serializeTupleData branch with a null element. --- .../internal/SerializerUtilsTest.java | 93 ++++++++++--------- 1 file changed, 51 insertions(+), 42 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java index e53d61166..265cd1cfb 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java @@ -159,52 +159,61 @@ private Object[][] nestedNullableData() throws Exception { UUID uuid = UUID.fromString("61f0c404-5cb3-11e7-907b-a6006ad3dba0"); InetAddress ipv4 = InetAddress.getByName("1.2.3.4"); return new Object[][] { - // A present Nullable element of each datatype nested in a Tuple, with a trailing - // non-nullable sibling whose bytes misalign if the present-marker is dropped. - {"Tuple(Nullable(String), String)", Arrays.asList("opt", "tail")}, - {"Tuple(Nullable(FixedString(3)), String)", Arrays.asList("abc", "tail")}, - {"Tuple(Nullable(Int8), String)", Arrays.asList((byte) -5, "tail")}, - {"Tuple(Nullable(UInt8), String)", Arrays.asList((short) 200, "tail")}, - {"Tuple(Nullable(Int16), String)", Arrays.asList((short) -1600, "tail")}, - {"Tuple(Nullable(UInt16), String)", Arrays.asList(40000, "tail")}, - {"Tuple(Nullable(Int32), String)", Arrays.asList(42, "tail")}, - {"Tuple(Nullable(UInt32), String)", Arrays.asList(4_000_000_000L, "tail")}, - {"Tuple(Nullable(Int64), String)", Arrays.asList(-64L, "tail")}, - {"Tuple(Nullable(UInt64), String)", Arrays.asList(BigInteger.valueOf(64), "tail")}, - {"Tuple(Nullable(Float32), String)", Arrays.asList(1.5f, "tail")}, - {"Tuple(Nullable(Float64), String)", Arrays.asList(2.5d, "tail")}, - {"Tuple(Nullable(Bool), String)", Arrays.asList(true, "tail")}, - {"Tuple(Nullable(UUID), String)", Arrays.asList(uuid, "tail")}, - {"Tuple(Nullable(Date), String)", Arrays.asList(LocalDate.of(2021, 2, 3), "tail")}, - {"Tuple(Nullable(Decimal64(4)), String)", Arrays.asList(new BigDecimal("1.2345"), "tail")}, - {"Tuple(Nullable(IPv4), String)", Arrays.asList(ipv4, "tail")}, + // Each present Nullable element sits in the MIDDLE of the schema: a non-nullable + // leading column, the Nullable, then a trailing non-nullable Float64. If the + // present-marker byte is dropped, every following byte shifts and the trailing + // Float64 reads a wrong value, so a faulty serialization is detected positionally + // rather than only by running out of bytes. The assertion compares the whole row. + {"Tuple(Int32, Nullable(String), Float64)", Arrays.asList(7, "opt", 9.5d)}, + {"Tuple(Int32, Nullable(FixedString(3)), Float64)", Arrays.asList(7, "abc", 9.5d)}, + {"Tuple(Int32, Nullable(Int8), Float64)", Arrays.asList(7, (byte) -5, 9.5d)}, + {"Tuple(Int32, Nullable(UInt8), Float64)", Arrays.asList(7, (short) 200, 9.5d)}, + {"Tuple(Int32, Nullable(Int16), Float64)", Arrays.asList(7, (short) -1600, 9.5d)}, + {"Tuple(Int32, Nullable(UInt16), Float64)", Arrays.asList(7, 40000, 9.5d)}, + {"Tuple(Int32, Nullable(Int32), Float64)", Arrays.asList(7, 42, 9.5d)}, + {"Tuple(Int32, Nullable(UInt32), Float64)", Arrays.asList(7, 4_000_000_000L, 9.5d)}, + {"Tuple(Int32, Nullable(Int64), Float64)", Arrays.asList(7, -64L, 9.5d)}, + {"Tuple(Int32, Nullable(UInt64), Float64)", Arrays.asList(7, BigInteger.valueOf(64), 9.5d)}, + {"Tuple(Int32, Nullable(Float32), Float64)", Arrays.asList(7, 1.5f, 9.5d)}, + {"Tuple(Int32, Nullable(Float64), Float64)", Arrays.asList(7, 2.5d, 9.5d)}, + {"Tuple(Int32, Nullable(Bool), Float64)", Arrays.asList(7, true, 9.5d)}, + {"Tuple(Int32, Nullable(UUID), Float64)", Arrays.asList(7, uuid, 9.5d)}, + {"Tuple(Int32, Nullable(Date), Float64)", Arrays.asList(7, LocalDate.of(2021, 2, 3), 9.5d)}, + {"Tuple(Int32, Nullable(Decimal64(4)), Float64)", Arrays.asList(7, new BigDecimal("1.2345"), 9.5d)}, + {"Tuple(Int32, Nullable(IPv4), Float64)", Arrays.asList(7, ipv4, 9.5d)}, // A Tuple value given as a Java array (not a List) takes the other branch of - // serializeTupleData, which is routed through the same nested-marker path. - {"Tuple(Nullable(String), String)", new Object[] {"opt", "tail"}}, - - // The same marker handling on the Map value path, across a range of widths. - {"Map(String, Nullable(String))", newMap("k", "v")}, - {"Map(String, Nullable(Int32))", newMap("k", 32)}, - {"Map(String, Nullable(Float64))", newMap("k", 2.5d)}, - {"Map(String, Nullable(UUID))", newMap("k", uuid)}, - - // Null elements/values still serialize a single null-marker byte. - {"Tuple(Nullable(String), String)", Arrays.asList(null, "tail")}, - {"Tuple(Nullable(Int32), String)", Arrays.asList(null, "tail")}, - {"Tuple(Nullable(Int32), Nullable(String))", Arrays.asList(null, null)}, - {"Map(String, Nullable(String))", newMap("k", null)}, + // serializeTupleData, which is routed through the same nested-marker path, for + // both a present value and a null. + {"Tuple(Int32, Nullable(String), Float64)", new Object[] {7, "opt", 9.5d}}, + {"Tuple(Int32, Nullable(String), Float64)", new Object[] {7, null, 9.5d}}, + + // The Map value path: the Nullable map value sits between the key and a trailing + // Float64, so a dropped value-marker misaligns the float. + {"Tuple(Int32, Map(String, Nullable(String)), Float64)", Arrays.asList(7, newMap("k", "v"), 9.5d)}, + {"Tuple(Int32, Map(String, Nullable(Int32)), Float64)", Arrays.asList(7, newMap("k", 32), 9.5d)}, + {"Tuple(Int32, Map(String, Nullable(Float64)), Float64)", Arrays.asList(7, newMap("k", 2.5d), 9.5d)}, + {"Tuple(Int32, Map(String, Nullable(UUID)), Float64)", Arrays.asList(7, newMap("k", uuid), 9.5d)}, + + // Null elements/values still serialize a single null-marker byte; the trailing + // Float64 confirms the following data stays aligned. + {"Tuple(Int32, Nullable(String), Float64)", Arrays.asList(7, null, 9.5d)}, + {"Tuple(Int32, Nullable(Int32), Nullable(String), Float64)", Arrays.asList(7, null, null, 9.5d)}, + {"Tuple(Int32, Map(String, Nullable(String)), Float64)", Arrays.asList(7, newMap("k", null), 9.5d)}, // Containers compose: marker handling threads through nested Tuple/Map/Array, - // including Array(Tuple(Nullable)) which is how Nested columns are encoded. - {"Tuple(String, Map(String, Nullable(String)))", Arrays.asList("id", newMap("k1", "v1", "k2", null))}, - {"Tuple(Nullable(String), Map(String, Nullable(Int32)))", Arrays.asList("opt", newMap("k", 7))}, - {"Array(Tuple(Nullable(String), String))", Arrays.asList(Arrays.asList("a", "b"), Arrays.asList(null, "c"))}, - {"Tuple(Array(Nullable(Int32)), String)", Arrays.asList(Arrays.asList(1, null, 3), "tail")}, - - // Contrast: non-nullable nested elements must keep serializing without a marker. - {"Tuple(Int32, String)", Arrays.asList(7, "tail")}, - {"Map(String, String)", newMap("k", "v")}, + // including Array(Tuple(Nullable)) which is how Nested columns are encoded. A + // trailing Float64 after each nested container detects misalignment. + {"Array(Tuple(Int32, Nullable(String), Float64))", + Arrays.asList(Arrays.asList(7, "a", 9.5d), Arrays.asList(7, null, 8.5d))}, + {"Tuple(String, Map(String, Nullable(Int32)), Float64)", + Arrays.asList("id", newMap("k1", 7, "k2", null), 9.5d)}, + {"Tuple(Array(Nullable(Int32)), Float64)", Arrays.asList(Arrays.asList(1, null, 3), 9.5d)}, + + // Contrast: non-nullable nested elements must keep serializing without a marker, + // so these rows round-trip identically with or without the fix. + {"Tuple(Int32, String, Float64)", Arrays.asList(7, "tail", 9.5d)}, + {"Tuple(Int32, Map(String, String), Float64)", Arrays.asList(7, newMap("k", "v"), 9.5d)}, }; }