diff --git a/Cargo.lock b/Cargo.lock index 3e1b0dbcd3a39..0850093f8147c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1116,8 +1116,8 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "h2 0.3.26", - "h2 0.4.12", + "h2 0.3.27", + "h2 0.4.17", "http 0.2.12", "http 1.4.2", "http-body 0.4.6", @@ -4176,9 +4176,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" dependencies = [ "bytes", "fnv", @@ -4195,9 +4195,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.12" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +checksum = "9f877e75f39e9827ec50a572dd592684ac28c029578726c85f1b2aa6ab807449" dependencies = [ "atomic-waker", "bytes", @@ -4552,7 +4552,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.26", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -4576,7 +4576,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.12", + "h2 0.4.17", "http 1.4.2", "http-body 1.0.1", "httparse", @@ -8014,7 +8014,6 @@ dependencies = [ "tokio", "tokio-metrics", "tokio-openssl", - "tokio-stream", "tokio-util", "tracing", "uuid", @@ -11371,7 +11370,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.12", + "h2 0.4.17", "http 1.4.2", "http-body 1.0.1", "http-body-util", @@ -13364,7 +13363,7 @@ dependencies = [ "base64 0.22.1", "bytes", "flate2", - "h2 0.4.12", + "h2 0.4.17", "http 1.4.2", "http-body 1.0.1", "http-body-util", diff --git a/bin/check-package-isolation b/bin/check-package-isolation index c8ee9c12b9d95..be1b83296015c 100755 --- a/bin/check-package-isolation +++ b/bin/check-package-isolation @@ -38,6 +38,14 @@ mapfile -t packages < <( shard_count="${BUILDKITE_PARALLEL_JOB_COUNT:-1}" shard_index="${BUILDKITE_PARALLEL_JOB:-0}" +# Populate the registry and git caches before any package is checked. Without +# this, whichever package runs first is the one that clones every git +# dependency, so a transient fetch failure there gets reported as that package +# failing to compile in isolation. Once the caches are warm the checks below +# need no network at all. +echo "--- :rust: cargo fetch" +retry cargo fetch || die "failed to fetch dependencies" + failed=() checked=0 for i in "${!packages[@]}"; do diff --git a/deny.toml b/deny.toml index 0b1f46fc16a98..1ce5b8f5d0566 100644 --- a/deny.toml +++ b/deny.toml @@ -295,6 +295,13 @@ ignore = [ "RUSTSEC-2026-0247", # `aws-sdk-s3` and `launchdarkly-server-sdk` both require `lru ^0.16.3` in their latest releases "RUSTSEC-2026-0253", + # `h2` unbounded empty DATA frames, only patched in `h2 >=0.4.16`. Our + # `h2 0.4` is patched, but `h2 0.3` is also in the tree via `hyper 0.14`, + # which `mz-aws-util` keeps to override the DNS resolver of the AWS SDK + # (`aws-smithy-runtime/connector-hyper-0-14-x`). The `h2 0.3` line has no + # patched release, so this cannot be resolved by a version bump. That + # client only talks to AWS endpoints, and the advisory is low severity. + "RUSTSEC-2026-0258", ] # Must be manually kept in sync with about.toml. diff --git a/src/environmentd/Cargo.toml b/src/environmentd/Cargo.toml index e32b3af4725ef..caed4dde55feb 100644 --- a/src/environmentd/Cargo.toml +++ b/src/environmentd/Cargo.toml @@ -104,7 +104,7 @@ tempfile = { workspace = true, optional = true } thiserror.workspace = true tokio.workspace = true tokio-postgres.workspace = true -tokio-stream = { workspace = true, features = ["net"] } +tokio-stream = { workspace = true, features = ["net"], optional = true } tokio-metrics.workspace = true tower.workspace = true tower-http.workspace = true @@ -172,6 +172,7 @@ test = [ "semver", "sentry-tracing", "tempfile", + "tokio-stream", "tracing-subscriber", "mz-postgres-util", "mz-tracing", diff --git a/src/pgwire/Cargo.toml b/src/pgwire/Cargo.toml index feeb5b661c3ea..06b0497ff8479 100644 --- a/src/pgwire/Cargo.toml +++ b/src/pgwire/Cargo.toml @@ -35,7 +35,6 @@ mz-sql = { path = "../sql" } openssl.workspace = true postgres.workspace = true tokio.workspace = true -tokio-stream.workspace = true tokio-openssl.workspace = true tokio-util = { workspace = true, features = ["codec"] } tokio-metrics.workspace = true diff --git a/test/sqllogictest/distinct_arrangements.slt b/test/sqllogictest/distinct_arrangements.slt index 475648745c97b..05d66f9d9a694 100644 --- a/test/sqllogictest/distinct_arrangements.slt +++ b/test/sqllogictest/distinct_arrangements.slt @@ -82,10 +82,12 @@ SELECT mz_unsafe.mz_sleep(4) query T SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_introspection.mz_dataflow_operators mdo ON mash.operator_id = mdo.id ORDER BY mdo.name; ---- +Arrange errors ArrangeBy[[Column(0), Column(1)]] ArrangeBy[[Column(0)]] Arranged DistinctBy Arranged DistinctBy +Distinct errors DistinctBy DistinctBy DistinctByErrorCheck @@ -120,6 +122,7 @@ SELECT mz_unsafe.mz_sleep(4) query T SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_introspection.mz_dataflow_operators mdo ON mash.operator_id = mdo.id ORDER BY mdo.name; ---- +Arrange errors ArrangeBy[[Column(0, "a"), Column(1, "b")]] ArrangeBy[[Column(0, "a"), Column(1, "b")]]-errors Arranged TopK input @@ -130,6 +133,7 @@ Arranged TopK input Arranged TopK input Arranged TopK input Arranged TopK input +Distinct errors Reduced TopK input Reduced TopK input Reduced TopK input @@ -218,6 +222,11 @@ SELECT mz_unsafe.mz_sleep(4) query T SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_introspection.mz_dataflow_operators mdo ON mash.operator_id = mdo.id ORDER BY mdo.name; ---- +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors Arranged TopK input Arranged TopK input Arranged TopK input @@ -242,6 +251,11 @@ Arranged TopK input Arranged TopK input Arranged TopK input Arranged TopK input +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors Reduced TopK input Reduced TopK input Reduced TopK input @@ -377,11 +391,18 @@ AccumulableErrorCheck Arrange ReduceMinsMaxes Arrange ReduceMinsMaxes Arrange ReduceMinsMaxes +Arrange bundle err +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors Arrange recursive err ArrangeAccumulable [val: empty] ArrangeBy[[Column(0, "user_id")]] ArrangeBy[[Column(0, "user_id")]] ArrangeBy[[Column(0, "user_id")]] +ArrangeBy[[Column(0, "user_id")]]-errors ArrangeBy[[Column(2, "user_id")]] Arranged DistinctBy Arranged MinsMaxesHierarchical input @@ -405,6 +426,13 @@ Arranged MinsMaxesHierarchical input Arranged MinsMaxesHierarchical input Arranged MinsMaxesHierarchical input Arranged MinsMaxesHierarchical input +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors[[Column(0), Column(1)]] +Distinct errors[[Column(0, "user_id")]] Distinct recursive err DistinctBy DistinctByErrorCheck @@ -451,7 +479,11 @@ query T SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_introspection.mz_dataflow_operators mdo ON mash.operator_id = mdo.id ORDER BY mdo.name; ---- AccumulableErrorCheck +Arrange bundle err +Arrange errors ArrangeAccumulable [val: empty] +Distinct errors +Distinct errors[[]] ReduceAccumulable statement ok @@ -475,7 +507,11 @@ SELECT mz_unsafe.mz_sleep(4) query T SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_introspection.mz_dataflow_operators mdo ON mash.operator_id = mdo.id ORDER BY mdo.name; ---- +Arrange errors +Arrange errors Arrange recursive err +Distinct errors +Distinct errors Distinct recursive err statement ok @@ -501,9 +537,11 @@ SELECT mz_unsafe.mz_sleep(4) query T SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_introspection.mz_dataflow_operators mdo ON mash.operator_id = mdo.id ORDER BY mdo.name; ---- +Arrange errors Arrange export iterative Arrange export iterative err Arrange recursive err +Distinct errors Distinct recursive err statement ok @@ -542,7 +580,11 @@ query T SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_introspection.mz_dataflow_operators mdo ON mash.operator_id = mdo.id ORDER BY mdo.name; ---- AccumulableErrorCheck +Arrange bundle err +Arrange errors ArrangeAccumulable [val: empty] +Distinct errors +Distinct errors[[]] ReduceAccumulable statement ok @@ -564,7 +606,9 @@ SELECT mz_unsafe.mz_sleep(4) query T SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_introspection.mz_dataflow_operators mdo ON mash.operator_id = mdo.id ORDER BY mdo.name; ---- +Arrange errors ArrangeBy[[Column(0)]] +Distinct errors Threshold local statement ok @@ -630,6 +674,13 @@ SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_intros ---- AccumulableErrorCheck AccumulableErrorCheck +Arrange bundle err +Arrange bundle err +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors ArrangeAccumulable [val: empty] ArrangeAccumulable [val: empty] ArrangeBy[[Column(0, "a"), Column(1, "b")]] @@ -646,6 +697,13 @@ ArrangeBy[[Column(0, "sum")]] ArrangeBy[[Column(0, "sum")]]-errors ArrangeBy[[]] ArrangeBy[[]] +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors[[]] +Distinct errors[[]] ReduceAccumulable ReduceAccumulable @@ -731,6 +789,24 @@ SELECT mz_unsafe.mz_sleep(4) query T SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_introspection.mz_dataflow_operators mdo ON mash.operator_id = mdo.id ORDER BY mdo.name; ---- +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors +Arrange errors ArrangeBy[[Column(0, "num")]] ArrangeBy[[Column(0, "num")]] ArrangeBy[[Column(0, "num")]] @@ -749,12 +825,34 @@ ArrangeBy[[Column(0, "num")]] ArrangeBy[[Column(0, "num")]] ArrangeBy[[Column(0, "num")]] ArrangeBy[[Column(0, "num")]] +ArrangeBy[[Column(0, "num")]]-errors +ArrangeBy[[Column(0, "num")]]-errors ArrangeBy[[]] ArrangeBy[[]] ArrangeBy[[]] ArrangeBy[[]] Arranged DistinctBy Arranged DistinctBy +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors[[Column(0, "num")]] +Distinct errors[[Column(0, "num")]] DistinctBy DistinctBy DistinctByErrorCheck @@ -790,10 +888,18 @@ query T SELECT mdo.name FROM mz_introspection.mz_arrangement_sharing mash JOIN mz_introspection.mz_dataflow_operators mdo ON mash.operator_id = mdo.id ORDER BY mdo.name; ---- AccumulableErrorCheck +Arrange bundle err +Arrange errors +Arrange errors ArrangeAccumulable [val: empty] ArrangeBy[[Column(0, "sum")]] ArrangeBy[[Column(0, "sum")]]-errors ArrangeBy[[]] +ArrangeBy[[]]-errors +Distinct errors +Distinct errors +Distinct errors[[]] +Distinct errors[[]] ReduceAccumulable statement ok @@ -862,10 +968,18 @@ AccumulableErrorCheck AccumulableErrorCheck Arrange ReduceMinsMaxes Arrange ReduceMinsMaxes +Arrange bundle err +Arrange bundle err +Arrange bundle err +Arrange errors +Arrange errors +Arrange errors +Arrange errors ArrangeAccumulable [val: empty] ArrangeAccumulable [val: empty] ArrangeAccumulable [val: empty] ArrangeBy[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(Row{[Int64(2)]}), ReprColumnType { scalar_type: Int64, nullable: false }))]] +ArrangeBy[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(Row{[Int64(2)]}), ReprColumnType { scalar_type: Int64, nullable: false }))]]-errors ArrangeBy[[CallUnary(CastInt32ToInt64(CastInt32ToInt64), Column(0, "id"))]] ArrangeBy[[Column(0)]] ArrangeBy[[Column(0, "data")]] @@ -899,6 +1013,14 @@ Arranged TopK input Arranged TopK input Arranged TopK input Arranged TopK input +Distinct errors +Distinct errors +Distinct errors +Distinct errors +Distinct errors[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(Row{[Int64(2)]}), ReprColumnType { scalar_type: Int64, nullable: false }))]] +Distinct errors[[Column(0)]] +Distinct errors[[Column(0)]] +Distinct errors[[]] ReduceAccumulable ReduceAccumulable ReduceAccumulable @@ -1058,9 +1180,11 @@ ORDER BY mdod.name ---- AccumulableErrorCheck 4 Arrange␠ReduceMinsMaxes 1 +Arrange␠errors 1 ArrangeAccumulable␠[val:␠empty] 4 ArrangeBy[[Column(0,␠"operator_id")]] 2 ArrangeBy[[Column(1,␠"dataflow_id")]] 2 +Distinct␠errors 1 ReduceAccumulable 4 ReduceMinsMaxes 1 ReduceMinsMaxes␠Error␠Check 1 diff --git a/test/sqllogictest/metric_sink.slt b/test/sqllogictest/metric_sink.slt index a03c57af4b0bf..b5b76ffdddf59 100644 --- a/test/sqllogictest/metric_sink.slt +++ b/test/sqllogictest/metric_sink.slt @@ -389,6 +389,64 @@ CREATE METRIC SINK rbac_ms FROM rbac_src WITH (PREFIX = 'mz_metric_sink_rbac_') ---- COMPLETE 0 +# TODO(linear#SQL-645): re-enable the block below once the sink operator skips a row whose +# `labels` map carries a null value. Today `extract_row` unwraps such a value as a string and +# panics, which aborts clusterd, so the test reproduces the crash instead of asserting the +# behavior it describes. +# +# # A `map[text=>text]` legally holds null *values*, and the shaping coalesces only +# # a null `labels` map as a whole, so such a row reaches the sink operator, which +# # has no label set it can represent and must skip it. Panicking instead aborts the +# # whole clusterd process, and the sink re-renders over the same persisted row on +# # restart, crash-looping every dataflow on the cluster. +# # +# # The same sink covers notice bookkeeping: `LiteralConstraints` attaches an "index +# # too wide" notice for the `metric_name = 'm'` filter over `nulls_idx`, keyed by +# # the sink's own id, and `DROP METRIC SINK` has to retract it again. +# +# statement ok +# CREATE TABLE nulls (metric_name text, metric_type text, labels map[text=>text], value double, help text) +# +# statement ok +# INSERT INTO nulls VALUES ('m', 'gauge', '{good=>1, bad=>NULL}'::map[text=>text], 1, 'h') +# +# statement ok +# CREATE INDEX nulls_idx ON nulls (metric_name, metric_type) +# +# statement ok +# CREATE VIEW nulls_v AS SELECT * FROM nulls WHERE metric_name = 'm' +# +# statement ok +# CREATE METRIC SINK ms_nulls FROM nulls_v WITH (PREFIX = 'mz_metric_sink_nulls_') +# +# # The replica folded that row and still answers: `count(*)` needs a dataflow on +# # `quickstart`, the cluster the sink runs on. +# +# query I +# SELECT count(*) FROM nulls +# ---- +# 1 +# +# # `mz_optimizer_notices` is readable only by `mz_monitor`, hence `mz_system`. +# +# simple conn=mz_system,user=mz_system +# SELECT count(*) FROM mz_internal.mz_optimizer_notices WHERE message LIKE '%nulls_idx%' +# ---- +# 1 +# COMPLETE 1 +# +# statement ok +# DROP METRIC SINK ms_nulls +# +# simple conn=mz_system,user=mz_system +# SELECT count(*) FROM mz_internal.mz_optimizer_notices WHERE message LIKE '%nulls_idx%' +# ---- +# 0 +# COMPLETE 1 +# +# statement ok +# DROP TABLE nulls CASCADE + # With the flag off the statement is refused, and the error says which flag. simple conn=mz_system,user=mz_system diff --git a/test/testdrive-old-kafka-src-syntax/kafka-avro-upsert-sinks.td b/test/testdrive-old-kafka-src-syntax/kafka-avro-upsert-sinks.td index 37cc283a7fcca..5cd035578cf14 100644 --- a/test/testdrive-old-kafka-src-syntax/kafka-avro-upsert-sinks.td +++ b/test/testdrive-old-kafka-src-syntax/kafka-avro-upsert-sinks.td @@ -429,3 +429,9 @@ contains:upsert key could not be validated as unique "Dataflow: materialize.public.input_with_deletions_keyed" "Reduced Fallibly MinsMaxesHierarchical" "Dataflow: materialize.public.input_with_deletions_keyed" "Reduced Fallibly MinsMaxesHierarchical" "Dataflow: materialize.public.input_with_deletions_keyed" "Reduced Fallibly MinsMaxesHierarchical" +"Dataflow: materialize.public.input_keyed" "Arrange errors" +"Dataflow: materialize.public.input_keyed" "Distinct errors" +"Dataflow: materialize.public.input_keyed_ab" "Arrange errors" +"Dataflow: materialize.public.input_keyed_ab" "Distinct errors" +"Dataflow: materialize.public.input_with_deletions_keyed" "Arrange errors" +"Dataflow: materialize.public.input_with_deletions_keyed" "Distinct errors" diff --git a/test/testdrive-old-kafka-src-syntax/monotonic.td b/test/testdrive-old-kafka-src-syntax/monotonic.td index df95f40743ba8..d2a19ac0de6bc 100644 --- a/test/testdrive-old-kafka-src-syntax/monotonic.td +++ b/test/testdrive-old-kafka-src-syntax/monotonic.td @@ -386,3 +386,11 @@ Target cluster: quickstart "Dataflow: materialize.public.monotonic_min" ReduceMonotonic "Dataflow: materialize.public.non_dbz_data_indexed_primary_idx" "ArrangeBy[[Column(0, \"a\"), Column(1, \"b\")]]" "Dataflow: materialize.public.non_dbz_data_indexed_primary_idx" "ArrangeBy[[Column(0, \"a\"), Column(1, \"b\")]]-errors" +"Dataflow: materialize.public.m1" "Arrange errors" +"Dataflow: materialize.public.m1" "Distinct errors" +"Dataflow: materialize.public.monotonic_fused" "Arrange errors" +"Dataflow: materialize.public.monotonic_fused" "Distinct errors" +"Dataflow: materialize.public.monotonic_max" "Arrange errors" +"Dataflow: materialize.public.monotonic_max" "Distinct errors" +"Dataflow: materialize.public.monotonic_min" "Arrange errors" +"Dataflow: materialize.public.monotonic_min" "Distinct errors" diff --git a/test/testdrive/metric-sink.td b/test/testdrive/metric-sink.td index 9aede34f74e60..870da103caef9 100644 --- a/test/testdrive/metric-sink.td +++ b/test/testdrive/metric-sink.td @@ -22,6 +22,14 @@ # the default timeout is too tight here. $ set-sql-timeout duration=60s +# `mz_cluster_prometheus_metrics` is a per-replica log source, so every read of it +# below has to name a replica. The sink runs on each replica of its cluster and +# each replica owns its own registry, so targeting one replica is also what keeps +# the series counts below independent of how many replicas `quickstart` has. `r1` +# is the default replica name, and the multi-replica testdrive configurations keep +# a replica of that name on `quickstart`. +> SET cluster_replica = r1 + > CREATE TABLE t (k text NOT NULL, val double) > INSERT INTO t VALUES ('a', 1), ('b', 2)