pyproject.toml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
[project]
name = "pyiceberg"
version = "0.12.0"
description = "Apache Iceberg is an open table format for huge analytic datasets"
authors = [{ name = "Apache Software Foundation", email = "dev@iceberg.apache.org" }]
requires-python = ">=3.10.0,<4.0.0"
readme = "README.md"
license = "Apache-2.0"
license-files = ["LICENSE", "NOTICE"]
classifiers = [
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
]
dependencies = [
    "mmh3>=4.0.0",
    "requests>=2.20.0",
    "click>=7.1.1",
    "rich>=10.11.0",
    "strictyaml>=1.7.0", # CVE-2020-14343 was fixed in 5.4.
    "pydantic>=2.0,<3.0,!=2.4.0,!=2.4.1,!=2.12.0,!=2.12.1", # 2.4.0, 2.4.1, 2.12.0, 2.12.1 has a critical bug
    "fsspec>=2023.1.0",
    "pyparsing>=3.1.0",
    "tenacity>=8.2.3",
    "pyroaring>=1.0.0",
    "cachetools>=5.5",
    "zstandard>=0.13.0"
]

[project.urls]
Homepage = "https://py.iceberg.apache.org/"
Repository = "https://github.com/apache/iceberg-python"

[project.scripts]
pyiceberg = "pyiceberg.cli.console:run"

[project.optional-dependencies]
pyarrow = [
    "pyarrow>=18.0.0",
]
pandas = [
    "pandas>=1.0.0",
    "pyarrow>=18.0.0",
]
duckdb = [
    "duckdb>=0.5.0",
    "pyarrow>=18.0.0",
]
ray = [
    "ray>=2.10.0",
    "pyarrow>=18.0.0",
    "pandas>=1.0.0",
]
bodo = ["bodo>=2025.7.4"]
daft = ["daft>=0.7.10"]
polars = ["polars>=1.21.0"]
snappy = ["python-snappy>=0.6.0"]
hive = ["thrift>=0.13.0"]
hive-kerberos = [
    "thrift>=0.13.0",
    "thrift-sasl>=0.4.3",
    "kerberos>=1.3.1",
]
s3fs = ["s3fs>=2023.1.0"]
glue = ["boto3>=1.24.59"]
adlfs = ["adlfs>=2024.7.0"]
dynamodb = ["boto3>=1.24.59"]
bigquery = ["google-cloud-bigquery>=3.33.0,<4"]
sql-postgres = [
    "sqlalchemy>=2.0.18,<3",
    "psycopg2-binary>=2.9.6",
]
sql-sqlite = ["sqlalchemy>=2.0.18,<3"]
gcsfs = ["gcsfs>=2023.1.0"]
rest-sigv4 = ["boto3>=1.24.59"]
hf = ["huggingface-hub>=0.24.0"]
# `pyiceberg-core` bundles a `datafusion-ffi` build, and the DataFusion FFI ABI is not stable
# across major versions (https://github.com/apache/datafusion/issues/17374). The two ranges below
# must name the same DataFusion major version: pyiceberg-core 0.10.x bundles datafusion-ffi 53.x.
pyiceberg-core = ["pyiceberg-core>=0.10.1,<0.11.0"]
datafusion = ["datafusion>=53,<54"]
gcp-auth = ["google-auth>=2.4.0"]
entra-auth = ["azure-identity>=1.25.1"]
geoarrow = ["geoarrow-pyarrow>=0.2.0"]

[dependency-groups]
dev = [
    "pytest==9.1.1",
    "pytest-checkdocs==2.14.0",
    "setuptools>=83.0.0",
    "prek>=0.2.1",
    "pytest-lazy-fixtures==1.4.1",
    "fastavro==1.12.2",
    "coverage[toml]>=7.4.2",
    "requests-mock==1.12.1",
    "moto[server]>=5.0.2",
    "typing-extensions==4.16.0",
    "pytest-mock==3.15.1",
    "pyspark[connect]==4.0.1",
    "protobuf==7.36.0", # match Spark Connect's gencode
    "cython>=3.0.0",
    "deptry>=0.14",
    "docutils!=0.21.post1",
    "mypy-boto3-glue>=1.28.18",
    "mypy-boto3-dynamodb>=1.28.18",
    "google-cloud-bigquery>=3.33.0,<4",
    "pyarrow-stubs>=20.0.0.20251107", # Remove when pyarrow >= 23.0.0 https://github.com/apache/arrow/pull/47609
    "sqlalchemy>=2.0.18,<3",
    "papermill>=2.6.0",
    "nbformat>=5.10.0",
    "ipykernel>=6.29.0",
]
# for mkdocs
docs = [
    "mkdocs==1.6.1",
    "griffe==2.2.0",
    "jinja2==3.1.6",
    "mkdocstrings==1.0.6",
    "mkdocstrings-python==2.0.7",
    "mkdocs-literate-nav==0.6.3",
    "mkdocs-autorefs==1.4.4",
    "mkdocs-gen-files==0.6.1",
    "mkdocs-material==9.7.7",
    "mkdocs-material-extensions==1.3.1",
    "mkdocs-section-index==0.3.12",
]
# for experimentation
notebook = [
    "jupyterlab>=4.0.0",
]

[tool.uv]
default-groups = [
    "dev",
]

[build-system]
requires = ["setuptools>=83.0.0", "wheel", "Cython>=3.0.0"]
build-backend = "setuptools.build_meta"

[tool.ruff]
src = ['pyiceberg','tests']

# Exclude a variety of commonly ignored directories.
exclude = [
    ".bzr",
    ".direnv",
    ".eggs",
    ".git",
    ".git-rewrite",
    ".hg",
    ".mypy_cache",
    ".nox",
    ".pants.d",
    ".pytype",
    ".ruff_cache",
    ".svn",
    ".tox",
    ".venv",
    "__pypackages__",
    "_build",
    "buck-out",
    "build",
    "dist",
    "node_modules",
    "venv",
]

# Ignore _all_ violations.
# Same as Black.
line-length = 130

[tool.ruff.lint]
select = [
    "E", # pycodestyle
    "W", # pycodestyle
    "F", # Pyflakes
    "B", # flake8-bugbear
    "PIE", # flake8-pie
    "C4", # flake8-comprehensions
    "I", # isort
    "UP", # pyupgrade
    "T100", # flake8-debugger
]
ignore = []

# Allow autofix for all enabled rules (when `--fix`) is provided.
fixable = ["ALL"]
unfixable = []

per-file-ignores = {}

# Allow unused variables when underscore-prefixed.
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"

[tool.ruff.lint.pyupgrade]
# Preserve types, even if a file imports `from __future__ import annotations`.
keep-runtime-typing = true

[tool.ruff.lint.isort]
detect-same-package = true
lines-between-types = 0
known-first-party = ["pyiceberg", "tests"]
section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]

[tool.ruff.format]
quote-style = "double"

[tool.pytest.ini_options]
testpaths = ["tests"]

markers = [
  "unmarked: marks a test as a unittest",
  "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint args)",
  "adls: marks a test as requiring access to adls compliant storage (use with --adls.account-name, --adls.account-key, and --adls.endpoint args)",
  "integration: marks integration tests against Apache Spark",
  "gcs: marks a test as requiring access to gcs compliant storage (use with --gs.token, --gs.project, and --gs.endpoint)",
  "benchmark: collection of tests to validate read/write performance before and after a change",
  "notebook: marks tests that execute Jupyter notebooks via papermill",
]

# Turns a warning into an error
filterwarnings = [
  "error",
  # Ignore Python version deprecation warning from google.api_core while we still support 3.10
  "ignore:You are using a Python version.*which Google will stop supporting:FutureWarning:google.api_core",
  # Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections
  "ignore:unclosed database in <sqlite3.Connection object*:ResourceWarning",
  # Ignore Ray subprocess cleanup warnings
  "ignore:unclosed file:ResourceWarning",
  "ignore:subprocess.*is still running:ResourceWarning",
  # Ignore google-crc32c C extension missing warning (common on Python 3.14+)
  "ignore:As the c extension couldn't be imported:RuntimeWarning:google_crc32c",
  # Ignore Spark 4.0.1 pandas conversion warning under pandas 3.0
  "ignore:The copy keyword is deprecated and will be removed in a future version.*",
]

[tool.mypy]
mypy_path = "python"
no_implicit_optional = true
namespace_packages = false
warn_redundant_casts = true
warn_unreachable = true
warn_unused_ignores = true
disallow_any_generics = true
disallow_untyped_defs = true
implicit_reexport = false
ignore_missing_imports = true

[tool.coverage.run]
source = ['pyiceberg/']