From 7137163e1bb88ab62a5192d4772f2d61c461e0ec Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Mon, 6 Jul 2026 18:51:16 +0200 Subject: [PATCH 1/9] perf: Use semi-join instead of `unique` for checking 1:N relationships --- dataframely/functional.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataframely/functional.py b/dataframely/functional.py index b099fe54..baabc713 100644 --- a/dataframely/functional.py +++ b/dataframely/functional.py @@ -84,9 +84,9 @@ def require_relationship_one_to_at_least_one( columns, filtered to ensure a 1:{1,N} relationship. """ if drop_duplicates: - return lhs.unique(on, keep="none").join(rhs.unique(on), on=on) + return lhs.unique(on, keep="none").join(rhs, on=on, how="semi") - return lhs.join(rhs.unique(on), on=on) + return lhs.join(rhs, on=on, how="semi") # ------------------------------------------------------------------------------------ # From 3574fd7ef87e8da92a496f950e9bd1ddc68bb4ee Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Mon, 6 Jul 2026 22:41:25 +0200 Subject: [PATCH 2/9] Fix benchmark --- tests/benches/test_collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/benches/test_collection.py b/tests/benches/test_collection.py index f3a1aa3b..0bf0cb5f 100644 --- a/tests/benches/test_collection.py +++ b/tests/benches/test_collection.py @@ -15,13 +15,13 @@ def partitioned_dataset(dataset: pl.DataFrame) -> dict[str, pl.DataFrame]: "elevation", "aspect", "slope", - idx=pl.int_range(pl.len(), dtype=pl.UInt32), + idx=pl.int_range(pl.len(), dtype=pl.UInt32).shuffle(), ), "second": dataset.select( "horizontal_distance_to_hydrology", "vertical_distance_to_hydrology", "horizontal_distance_to_roadways", - idx=pl.int_range(pl.len(), dtype=pl.UInt32), + idx=pl.int_range(pl.len(), dtype=pl.UInt32).shuffle(), ), } From a5c02029c12b0c4b2c770b48ecf65bc51f883c78 Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Mon, 6 Jul 2026 23:15:29 +0200 Subject: [PATCH 3/9] perf: Leverage `collect_all` for eager collection validation & filtering --- dataframely/_polars.py | 17 ++++++++++---- dataframely/collection/collection.py | 33 +++++++++++++--------------- dataframely/schema.py | 4 ++-- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/dataframely/_polars.py b/dataframely/_polars.py index b19bb50f..cd58edb9 100644 --- a/dataframely/_polars.py +++ b/dataframely/_polars.py @@ -41,9 +41,18 @@ def timedelta_matches_resolution(d: dt.timedelta, resolution: str) -> bool: return datetime_matches_resolution(EPOCH_DATETIME + d, resolution) -def collect_if(lf: pl.LazyFrame, condition: bool) -> pl.DataFrame | pl.LazyFrame: - """Collect a lazy frame if the original was eager, otherwise return the lazy - frame.""" +def collect_if(lf: pl.LazyFrame, condition: bool) -> pl.LazyFrame: + """Collect a lazy frame based on `condition`.""" if condition: - return lf.collect() + return lf.collect().lazy() return lf + + +def collect_all_if( + lfs: dict[str, pl.LazyFrame], condition: bool +) -> dict[str, pl.LazyFrame]: + """Collect the lazy frames in the dictionary based on `condition`.""" + if condition: + dfs = pl.collect_all(lfs.values()) + return {k: v.lazy() for k, v in zip(lfs.keys(), dfs)} + return lfs diff --git a/dataframely/collection/collection.py b/dataframely/collection/collection.py index 059d52b6..83f7dfb9 100644 --- a/dataframely/collection/collection.py +++ b/dataframely/collection/collection.py @@ -31,7 +31,7 @@ from dataframely._filter import Filter from dataframely._native import format_rule_failures from dataframely._plugin import all_rules_required -from dataframely._polars import FrameType, collect_if +from dataframely._polars import FrameType, collect_all_if from dataframely._serialization import ( SERIALIZATION_FORMAT_VERSION, SchemaJSONDecoder, @@ -605,28 +605,25 @@ class HospitalInvoiceData(dy.Collection): result_cls = cls._init(results) primary_key = cls.common_primary_key() - keep: dict[str, pl.LazyFrame] = {} - for name, filter in filters.items(): - keep[name] = ( - filter.logic(result_cls) - .select(primary_key) - .pipe(collect_if, eager) - .lazy() - ) + keep = { + name: filter.logic(result_cls).select(primary_key) + for name, filter in filters.items() + } + keep = collect_all_if(keep, eager) - drop: dict[str, pl.LazyFrame] = {} - for failure_propagating_member in failure_propagating_members: - annotation_column = f"{failure_propagating_member}|failure_propagation" - drop[annotation_column] = ( + drop: dict[str, pl.LazyFrame] = { + f"{failure_propagating_member}|failure_propagation": ( failures[failure_propagating_member] ._lf.select(primary_key) .unique() - .pipe(collect_if, eager) - .lazy() ) + for failure_propagating_member in failure_propagating_members + } + drop = collect_all_if(drop, eager) # Now we can iterate over the results and left-join onto each individual - # filter to obtain independent boolean indicators of whether to keep the row + # filter to obtain independent boolean indicators of whether to keep the row. + lfs_with_eval: dict[str, pl.LazyFrame] = {} for member_name, filtered in results.items(): member_info = cls.members()[member_name] if member_info.ignored_in_filters: @@ -648,8 +645,8 @@ class HospitalInvoiceData(dy.Collection): maintain_order="left", ).with_columns(pl.col(name).fill_null(True)) - lf_with_eval = lf_with_eval.pipe(collect_if, eager).lazy() - + lfs_with_eval = collect_all_if(lfs_with_eval, eager) + for member_name, lf_with_eval in lfs_with_eval.items(): # Filtering `lf_with_eval` by the rows for which all joins # "succeeded", we can identify the rows that pass all the filters. We # keep these rows for the result. diff --git a/dataframely/schema.py b/dataframely/schema.py index c7fc92c4..726aa6dc 100644 --- a/dataframely/schema.py +++ b/dataframely/schema.py @@ -754,8 +754,8 @@ def filter( match_to_schema, cls, casting=("lenient" if cast else "none") ) if rules := cls._validation_rules(with_cast=cast): - evaluated = ( - lf.pipe(cls._with_evaluated_rules, rules).pipe(collect_if, eager).lazy() + evaluated = lf.pipe(cls._with_evaluated_rules, rules).pipe( + collect_if, eager ) filtered = evaluated.filter(pl.col(_COLUMN_VALID)).select( cls.column_names() From 3e0cff521b1a5e7d8407a26497043aef95a96eec Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Mon, 6 Jul 2026 23:27:03 +0200 Subject: [PATCH 4/9] perf: Allow specifying the engine for `validate` and `filter` --- dataframely/_polars.py | 10 ++++----- dataframely/collection/collection.py | 32 +++++++++++++++++++++------- dataframely/schema.py | 23 ++++++++++++-------- tests/benches/test_collection.py | 14 ++++++++---- 4 files changed, 53 insertions(+), 26 deletions(-) diff --git a/dataframely/_polars.py b/dataframely/_polars.py index cd58edb9..e8c83ce3 100644 --- a/dataframely/_polars.py +++ b/dataframely/_polars.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: BSD-3-Clause import datetime as dt -from typing import TypeVar +from typing import Any, TypeVar import polars as pl from polars.datatypes import DataTypeClass @@ -41,18 +41,18 @@ def timedelta_matches_resolution(d: dt.timedelta, resolution: str) -> bool: return datetime_matches_resolution(EPOCH_DATETIME + d, resolution) -def collect_if(lf: pl.LazyFrame, condition: bool) -> pl.LazyFrame: +def collect_if(lf: pl.LazyFrame, condition: bool, **kwargs: Any) -> pl.LazyFrame: """Collect a lazy frame based on `condition`.""" if condition: - return lf.collect().lazy() + return lf.collect(**kwargs).lazy() return lf def collect_all_if( - lfs: dict[str, pl.LazyFrame], condition: bool + lfs: dict[str, pl.LazyFrame], condition: bool, **kwargs: Any ) -> dict[str, pl.LazyFrame]: """Collect the lazy frames in the dictionary based on `condition`.""" if condition: - dfs = pl.collect_all(lfs.values()) + dfs = pl.collect_all(lfs.values(), **kwargs) return {k: v.lazy() for k, v in zip(lfs.keys(), dfs)} return lfs diff --git a/dataframely/collection/collection.py b/dataframely/collection/collection.py index 83f7dfb9..843e598f 100644 --- a/dataframely/collection/collection.py +++ b/dataframely/collection/collection.py @@ -380,7 +380,13 @@ def _preprocess_sample( @classmethod def validate( - cls, data: Mapping[str, FrameType], /, *, cast: bool = False, eager: bool = True + cls, + data: Mapping[str, FrameType], + /, + *, + cast: bool = False, + eager: bool = True, + **kwargs: Any, ) -> Self: """Validate that a set of data frames satisfy the collection's invariants. @@ -400,6 +406,8 @@ def validate( :meth:`~polars.LazyFrame.collect` on the individual member or :meth:`collect_all` on the collection. Note that, in the latter case, information from error messages is limited. + kwargs: Keyword arguments passed directly to :meth:`polars.collect_all` when + `eager=True`. Raises: ValueError: If an insufficient set of input data frames is provided, i.e. if @@ -421,7 +429,7 @@ def validate( if eager: # If we perform the validation eagerly, we call filter and check the failure # information to properly construct a useful error message. - filtered, failures = cls.filter(data, cast=cast, eager=True) + filtered, failures = cls.filter(data, cast=cast, eager=True, **kwargs) if any(len(failure) > 0 for failure in failures.values()): errors: dict[str, str] = {} for member, failure in failures.items(): @@ -535,7 +543,13 @@ def is_valid(cls, data: Mapping[str, FrameType], /, *, cast: bool = False) -> bo @classmethod def filter( - cls, data: Mapping[str, FrameType], /, *, cast: bool = False, eager: bool = True + cls, + data: Mapping[str, FrameType], + /, + *, + cast: bool = False, + eager: bool = True, + **kwargs: Any, ) -> CollectionFilterResult[Self]: """Filter the members data frame by their schemas and the collection's filters. @@ -551,6 +565,8 @@ def filter( eager: Whether the filter operation should be performed eagerly. Note that until https://github.com/pola-rs/polars/pull/24129 is released, eagerly filtering can provide significant speedups. + kwargs: Keyword arguments passed directly to :meth:`polars.collect_all` when + `eager=True`. Returns: A named tuple with fields `result` and `failure`. The `result` field @@ -609,7 +625,7 @@ class HospitalInvoiceData(dy.Collection): name: filter.logic(result_cls).select(primary_key) for name, filter in filters.items() } - keep = collect_all_if(keep, eager) + keep = collect_all_if(keep, eager, **kwargs) drop: dict[str, pl.LazyFrame] = { f"{failure_propagating_member}|failure_propagation": ( @@ -619,7 +635,7 @@ class HospitalInvoiceData(dy.Collection): ) for failure_propagating_member in failure_propagating_members } - drop = collect_all_if(drop, eager) + drop = collect_all_if(drop, eager, **kwargs) # Now we can iterate over the results and left-join onto each individual # filter to obtain independent boolean indicators of whether to keep the row. @@ -645,7 +661,7 @@ class HospitalInvoiceData(dy.Collection): maintain_order="left", ).with_columns(pl.col(name).fill_null(True)) - lfs_with_eval = collect_all_if(lfs_with_eval, eager) + lfs_with_eval = collect_all_if(lfs_with_eval, eager, **kwargs) for member_name, lf_with_eval in lfs_with_eval.items(): # Filtering `lf_with_eval` by the rows for which all joins # "succeeded", we can identify the rows that pass all the filters. We @@ -710,7 +726,7 @@ class HospitalInvoiceData(dy.Collection): result = CollectionFilterResult(cls._init(results), failures) if eager: - return result.collect_all() + return result.collect_all(**kwargs) return result def join( @@ -815,7 +831,7 @@ def collect_all(self) -> Self: The same collection with all members collected once. Members annotated with :class:`~dataframely.DataFrame` are returned as DataFrames, while members annotated with :class:`~dataframely.LazyFrame` are returned as - "shallow-lazy" frames (obtained by calling ``.collect().lazy()``). + "shallow-lazy" frames (obtained by calling `.collect().lazy()`). """ lazy_dict = self.to_dict() dfs = pl.collect_all(lazy_dict.values()) diff --git a/dataframely/schema.py b/dataframely/schema.py index 726aa6dc..19cb23e1 100644 --- a/dataframely/schema.py +++ b/dataframely/schema.py @@ -501,6 +501,7 @@ def validate( *, cast: bool = False, eager: Literal[True] = True, + **kwargs: Any, ) -> DataFrame[Self]: ... @overload @@ -512,6 +513,7 @@ def validate( *, cast: bool = False, eager: Literal[False], + **kwargs: Any, ) -> LazyFrame[Self]: ... @overload @@ -523,6 +525,7 @@ def validate( *, cast: bool = False, eager: bool, + **kwargs: Any, ) -> DataFrame[Self] | LazyFrame[Self]: ... @classmethod @@ -533,6 +536,7 @@ def validate( *, cast: bool = False, eager: bool = True, + **kwargs: Any, ) -> DataFrame[Self] | LazyFrame[Self]: """Validate that a data frame satisfies the schema. @@ -554,6 +558,7 @@ def validate( not surface *all* validation issues as the validation is aborted once the first failure is encountered. Likewise, the reported validation failure can be non-deterministic. + kwargs: Keyword arguments passed directly to :meth:`collect` when `eager=True`. Returns: The input eager or lazy frame, wrapped in a generic version of the @@ -574,7 +579,7 @@ def validate( `eager=False`. """ if eager: - out, failure = cls.filter(df, cast=cast, eager=True) + out, failure = cls.filter(df, cast=cast, eager=True, **kwargs) if len(failure) > 0: counts = failure.counts() raise ValidationError( @@ -699,6 +704,7 @@ def filter( *, cast: bool = False, eager: bool = True, + **kwargs: Any, ) -> FilterResult[Self] | LazyFilterResult[Self]: """Filter the data frame by the rules of this schema, returning `(valid, failures)`. @@ -709,16 +715,15 @@ def filter( succeeds. Args: - df: The data frame to filter for valid rows. - The data frame is collected within this method, regardless of whether - a :class:`~polars.DataFrame` or :class:`~polars.LazyFrame` is passed. - cast: - Whether columns with a wrong data type in the input data frame are + df: The data frame to filter for valid rows. The data frame is collected + within this method, regardless of whether a :class:`~polars.DataFrame` + or :class:`~polars.LazyFrame` is passed. + cast: Whether columns with a wrong data type in the input data frame are cast to the schema's defined data type if possible. Rows for which the cast fails for any column are filtered out. eager: Whether the filter operation should be performed eagerly. If `False`, the - returned lazy frame will - fail to collect if the validation does not pass. + returned lazy frame will fail to collect if the validation does not pass. + kwargs: Keyword arguments passed directly to :meth:`collect` when `eager=True`. Returns: A tuple of the validated rows in the input data frame (potentially @@ -755,7 +760,7 @@ def filter( ) if rules := cls._validation_rules(with_cast=cast): evaluated = lf.pipe(cls._with_evaluated_rules, rules).pipe( - collect_if, eager + collect_if, eager, **kwargs ) filtered = evaluated.filter(pl.col(_COLUMN_VALID)).select( cls.column_names() diff --git a/tests/benches/test_collection.py b/tests/benches/test_collection.py index 0bf0cb5f..a28ca58c 100644 --- a/tests/benches/test_collection.py +++ b/tests/benches/test_collection.py @@ -95,18 +95,24 @@ def one_to_at_least_one_reverse(self) -> pl.LazyFrame: @pytest.mark.benchmark(group="collection-filter-multi") +@pytest.mark.parametrize("engine", ["in-memory", "streaming"]) def test_multi_filter_validate( - benchmark: BenchmarkFixture, partitioned_dataset: dict[str, pl.DataFrame] + benchmark: BenchmarkFixture, + partitioned_dataset: dict[str, pl.DataFrame], + engine: str, ) -> None: - benchmark(MultiFilterCollection.validate, partitioned_dataset) + benchmark(MultiFilterCollection.validate, partitioned_dataset, engine=engine) @pytest.mark.benchmark(group="collection-filter-multi") +@pytest.mark.parametrize("engine", ["in-memory", "streaming"]) def test_multi_filter_filter( - benchmark: BenchmarkFixture, partitioned_dataset: dict[str, pl.DataFrame] + benchmark: BenchmarkFixture, + partitioned_dataset: dict[str, pl.DataFrame], + engine: str, ) -> None: def benchmark_fn() -> None: - _, failure = MultiFilterCollection.filter(partitioned_dataset) + _, failure = MultiFilterCollection.filter(partitioned_dataset, engine=engine) _ = [len(f) for f in failure.values()] benchmark(benchmark_fn) From 86109d20ccd4f5c0b56cf66ac50297201ad0c828 Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Mon, 6 Jul 2026 23:33:05 +0200 Subject: [PATCH 5/9] Fix --- dataframely/collection/collection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dataframely/collection/collection.py b/dataframely/collection/collection.py index 83f7dfb9..99c5db8b 100644 --- a/dataframely/collection/collection.py +++ b/dataframely/collection/collection.py @@ -647,6 +647,8 @@ class HospitalInvoiceData(dy.Collection): lfs_with_eval = collect_all_if(lfs_with_eval, eager) for member_name, lf_with_eval in lfs_with_eval.items(): + member_info = cls.members()[member_name] + # Filtering `lf_with_eval` by the rows for which all joins # "succeeded", we can identify the rows that pass all the filters. We # keep these rows for the result. From f7b80cececb11c2795ea1cbe7905c0a76d2909cf Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Mon, 6 Jul 2026 23:47:17 +0200 Subject: [PATCH 6/9] Fix --- dataframely/collection/collection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dataframely/collection/collection.py b/dataframely/collection/collection.py index 99c5db8b..2c8a5dc7 100644 --- a/dataframely/collection/collection.py +++ b/dataframely/collection/collection.py @@ -645,6 +645,8 @@ class HospitalInvoiceData(dy.Collection): maintain_order="left", ).with_columns(pl.col(name).fill_null(True)) + lfs_with_eval[member_name] = lf_with_eval + lfs_with_eval = collect_all_if(lfs_with_eval, eager) for member_name, lf_with_eval in lfs_with_eval.items(): member_info = cls.members()[member_name] From 3bf20b060845a0dccfe0df4f9e0ba4655183d505 Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Tue, 7 Jul 2026 02:19:37 +0200 Subject: [PATCH 7/9] More kwargs --- dataframely/collection/collection.py | 33 +++++++++++++++++----------- dataframely/schema.py | 13 ++++++----- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/dataframely/collection/collection.py b/dataframely/collection/collection.py index 2f0b4f16..b8d62478 100644 --- a/dataframely/collection/collection.py +++ b/dataframely/collection/collection.py @@ -406,8 +406,8 @@ def validate( :meth:`~polars.LazyFrame.collect` on the individual member or :meth:`collect_all` on the collection. Note that, in the latter case, information from error messages is limited. - kwargs: Keyword arguments passed directly to :meth:`polars.collect_all` when - `eager=True`. + kwargs: Keyword arguments passed directly to :meth:`polars.collect_all` and + :meth:`polars.LazyFrame.collect` when `eager=True`. Raises: ValueError: If an insufficient set of input data frames is provided, i.e. if @@ -497,7 +497,9 @@ def validate( return cls._init(members) @classmethod - def is_valid(cls, data: Mapping[str, FrameType], /, *, cast: bool = False) -> bool: + def is_valid( + cls, data: Mapping[str, FrameType], /, *, cast: bool = False, **kwargs: Any + ) -> bool: """Utility method to check whether :meth:`validate` raises an exception. Args: @@ -505,7 +507,9 @@ def is_valid(cls, data: Mapping[str, FrameType], /, *, cast: bool = False) -> bo dictionary must contain exactly one entry per member with the name of the member as key. cast: Whether columns with a wrong data type in the member data frame are - cast to their schemas' defined data types if possible. + cast to their schemas' defined data types if possible.] + kwargs: Keyword arguments passed directly to :meth:`polars.collect_all` and + :meth:`polars.LazyFrame.collect`. Returns: Whether the provided members satisfy the invariants of the collection. @@ -520,7 +524,7 @@ def is_valid(cls, data: Mapping[str, FrameType], /, *, cast: bool = False) -> bo members: dict[str, pl.LazyFrame] = {} for member, schema in cls.member_schemas().items(): if member in data: - if not schema.is_valid(data[member], cast=cast): + if not schema.is_valid(data[member], cast=cast, **kwargs): return False members[member] = data[member].lazy() @@ -531,9 +535,12 @@ def is_valid(cls, data: Mapping[str, FrameType], /, *, cast: bool = False) -> bo keep = [filter.logic(result_cls).select(primary_key) for filter in filters] joined = _join_all(*keep, on=primary_key, how="inner") removed_rows = pl.collect_all( - data[member].lazy().join(joined, on=primary_key, how="anti") - for member in cls.members() - if member in data + ( + data[member].lazy().join(joined, on=primary_key, how="anti") + for member in cls.members() + if member in data + ), + **kwargs, ) return all(df.is_empty() for df in removed_rows) @@ -565,8 +572,8 @@ def filter( eager: Whether the filter operation should be performed eagerly. Note that until https://github.com/pola-rs/polars/pull/24129 is released, eagerly filtering can provide significant speedups. - kwargs: Keyword arguments passed directly to :meth:`polars.collect_all` when - `eager=True`. + kwargs: Keyword arguments passed directly to :meth:`polars.collect_all` and + :meth:`polars.LazyFrame.collect` when `eager=True`. Returns: A named tuple with fields `result` and `failure`. The `result` field @@ -609,7 +616,7 @@ class HospitalInvoiceData(dy.Collection): continue member_result, failures[member_name] = member.schema.filter( - data[member_name].lazy(), cast=cast, eager=eager + data[member_name].lazy(), cast=cast, eager=eager, **kwargs ) results[member_name] = member_result.lazy() @@ -651,14 +658,14 @@ class HospitalInvoiceData(dy.Collection): filter_keep.lazy().with_columns(pl.lit(True).alias(name)), on=primary_key, how="left", - maintain_order="left", + # maintain_order="left", ).with_columns(pl.col(name).fill_null(False)) for name, filter_drop in drop.items(): lf_with_eval = lf_with_eval.join( filter_drop.with_columns(pl.lit(False).alias(name)), on=primary_key, how="left", - maintain_order="left", + # maintain_order="left", ).with_columns(pl.col(name).fill_null(True)) lfs_with_eval[member_name] = lf_with_eval diff --git a/dataframely/schema.py b/dataframely/schema.py index 19cb23e1..9d2e04e8 100644 --- a/dataframely/schema.py +++ b/dataframely/schema.py @@ -558,7 +558,8 @@ def validate( not surface *all* validation issues as the validation is aborted once the first failure is encountered. Likewise, the reported validation failure can be non-deterministic. - kwargs: Keyword arguments passed directly to :meth:`collect` when `eager=True`. + kwargs: Keyword arguments passed directly to :meth:`polars.LazyFrame.collect` + when `eager=True`. Returns: The input eager or lazy frame, wrapped in a generic version of the @@ -613,7 +614,7 @@ def validate( @classmethod def is_valid( - cls, df: pl.DataFrame | pl.LazyFrame, /, *, cast: bool = False + cls, df: pl.DataFrame | pl.LazyFrame, /, *, cast: bool = False, **kwargs: Any ) -> bool: """Check whether a data frame satisfies the schema. @@ -632,6 +633,7 @@ def is_valid( cast to the schema's defined data type before running validation. If set to `False`, a wrong data type will result in a return value of `False`. + kwargs: Keyword arguments passed directly to :meth:`polars.LazyFrame.collect`. Returns: Whether the provided dataframe can be validated with this schema. @@ -651,12 +653,12 @@ def is_valid( return ( lf.pipe(with_evaluation_rules, rules) .select(all_rules(rules.keys())) - .collect() + .collect(**kwargs) .item() ) # NOTE: We cannot simply return `True` here as, otherwise, we wouldn't # validate the schema. - return lf.select(pl.lit(True)).collect().item() + return lf.select(pl.lit(True)).collect(**kwargs).item() except SchemaError: # If we encounter a schema error, we gracefully handle this as 'invalid' return False @@ -723,7 +725,8 @@ def filter( cast fails for any column are filtered out. eager: Whether the filter operation should be performed eagerly. If `False`, the returned lazy frame will fail to collect if the validation does not pass. - kwargs: Keyword arguments passed directly to :meth:`collect` when `eager=True`. + kwargs: Keyword arguments passed directly to :meth:`polars.LazyFrame.collect` + when `eager=True`. Returns: A tuple of the validated rows in the input data frame (potentially From 74a152d194084687de10090dacdc2dd40fca5741 Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Tue, 7 Jul 2026 10:12:51 +0200 Subject: [PATCH 8/9] Update dataframely/collection/collection.py Co-authored-by: ElisabethBrockhausQC <125682683+ElisabethBrockhausQC@users.noreply.github.com> --- dataframely/collection/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataframely/collection/collection.py b/dataframely/collection/collection.py index b8d62478..2b2bbb0b 100644 --- a/dataframely/collection/collection.py +++ b/dataframely/collection/collection.py @@ -507,7 +507,7 @@ def is_valid( dictionary must contain exactly one entry per member with the name of the member as key. cast: Whether columns with a wrong data type in the member data frame are - cast to their schemas' defined data types if possible.] + cast to their schemas' defined data types if possible. kwargs: Keyword arguments passed directly to :meth:`polars.collect_all` and :meth:`polars.LazyFrame.collect`. From 705cc3ad0a3b40a1680ed2a2f5a89128df4a0950 Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Tue, 7 Jul 2026 10:17:38 +0200 Subject: [PATCH 9/9] Revert maintain order --- dataframely/collection/collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataframely/collection/collection.py b/dataframely/collection/collection.py index 2fced8cb..447909b0 100644 --- a/dataframely/collection/collection.py +++ b/dataframely/collection/collection.py @@ -658,14 +658,14 @@ class HospitalInvoiceData(dy.Collection): filter_keep.lazy().with_columns(pl.lit(True).alias(name)), on=primary_key, how="left", - # maintain_order="left", + maintain_order="left", ).with_columns(pl.col(name).fill_null(False)) for name, filter_drop in drop.items(): lf_with_eval = lf_with_eval.join( filter_drop.with_columns(pl.lit(False).alias(name)), on=primary_key, how="left", - # maintain_order="left", + maintain_order="left", ).with_columns(pl.col(name).fill_null(True)) lfs_with_eval[member_name] = lf_with_eval