pyproject.toml

[project]
name = "virtualizarr"
description = "Create virtual Zarr stores from archival data using xarray API"
readme = "README.md"
authors = [
    {name = "Tom Nicholas", email = "tom@earthmover.io"},
    {name = "Max Jones", email = "max@developmentseed.org"},
    {name = "Raphael Hagen", email = "raphael@carbonplan.org"},
]
license = "Apache-2.0"
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Science/Research",
    "Topic :: Scientific/Engineering",
    "Operating System :: OS Independent",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
]
requires-python = ">=3.11"
dynamic = ["version"]
dependencies = [
    "xarray>=2025.06.0",  # first version compatible with zarr 3.1.x fill value dtype handling
    "numpy>=2.1.0",  # first version with Python 3.13 wheels
    "universal-pathlib",
    "numcodecs>=0.15.1",
    "ujson",
    "packaging",
    "zarr>=3.1.0",
    "obstore>=0.7.0",
    "obspec_utils>=0.9.0",
]

# Dependency sets under optional-dependencies are available via PyPI
[project.optional-dependencies]
# for creating virtual datasets from objects on remote storage
remote = [
    "fsspec",
    "requests",
    "aiohttp",
    "s3fs",
]

# non-kerchunk-based parsers
hdf = [
    "h5py",
    "hdf5plugin",
    "imagecodecs",
    "imagecodecs-numcodecs==2024.6.1",
]

zarr = ["arro3-core"]

# kerchunk-based parsers
netcdf3 = [
    "virtualizarr[remote]",
    "kerchunk>=0.2.8",
    "scipy",
]
fits = [
    "virtualizarr[remote]",
    "kerchunk>=0.2.8",
    "astropy",
]
tiff = [
    "virtual-tiff",
]
kerchunk_parquet = [
   "virtualizarr[remote]",
   "fastparquet",
]

# kerchunk
all_parsers = [
    "virtualizarr[hdf]",
    "virtualizarr[netcdf3]",
    "virtualizarr[fits]",
    "virtualizarr[kerchunk_parquet]",
    "virtualizarr[tiff]",
    "virtualizarr[zarr]"
]

# writers
icechunk = [
    "icechunk>=1.1.2",
]


kerchunk = ["fastparquet", "pandas"]

all_writers = [
    "virtualizarr[icechunk]",
    "virtualizarr[kerchunk]",
]

# Dependency sets under dependencies-groups are NOT available via PyPI
[dependency-groups]
upstream = [
    'xarray @ git+https://github.com/pydata/xarray',
    'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib',
    'numcodecs @ git+https://github.com/zarr-developers/numcodecs',
    'ujson @ git+https://github.com/ultrajson/ultrajson',
    'zarr @ git+https://github.com/zarr-developers/zarr-python',
    'obspec_utils @ git+https://github.com/virtual-zarr/obspec-utils',
    # optional dependencies
    'astropy @ git+https://github.com/astropy/astropy',
    's3fs @ git+https://github.com/fsspec/s3fs',
    'kerchunk @ git+https://github.com/fsspec/kerchunk',
    'icechunk @ git+https://github.com/earth-mover/icechunk#subdirectory=icechunk-python',
    'virtual_tiff @ git+https://github.com/virtual-zarr/virtual-tiff',
]
docs = [
    "mkdocs-material[imaging]>=9.6.14",
    "mkdocs>=1.6.1",
    "mkdocstrings>=0.29.1",
    "mkdocstrings-python>=1.16.10",
    "markdown-exec[ansi]",
    "ruff",
]
dev = [
    "codecov",
    "mypy",
    "pandas-stubs",
    "pooch",
    "pre-commit",
    "pytest",
    "pytest-asyncio",
    "pytest-cov",
    "pytest-mypy",
    "pytest-timeout",
    "pytest-xdist",
    "ruff",
    "s3fs",
    "lithops",
    "dask",
]

[project.urls]
Home = "https://github.com/zarr-developers/VirtualiZarr"
Documentation = "https://github.com/zarr-developers/VirtualiZarr/blob/main/README.md"

[build-system]
requires = [
    "setuptools>=61.0.0",
    "wheel",
    "setuptools_scm[toml]>=7.0",
    "check-manifest",
]

[tool.pixi.workspace]
channels = ["conda-forge"]
platforms = ["linux-64", "osx-64", "osx-arm64"]

# Add the editable project pip install to the default group
[tool.pixi.pypi-dependencies]
virtualizarr = { path = ".", editable = true }

# Define a feature set for Python 3.11
[tool.pixi.feature.py311.dependencies]
python = "3.11.*"

# Define a feature set for Python 3.12
[tool.pixi.feature.py312.dependencies]
python = "3.12.*"

# Define a feature set for Python 3.13
[tool.pixi.feature.py313.dependencies]
python = "3.13.*"

# Define a feature set for Python 3.14
[tool.pixi.feature.py314.dependencies]
python = "3.14.*"

# Define a feature set for S3 testing with MinIO
[tool.pixi.feature.minio.pypi-dependencies]
"docker" = "*"
"minio" = "*"

# Install NetCDF and HDF5 from conda-forge for extra plugin capabilities
[tool.pixi.feature.hdf5-lib.dependencies]
netcdf4 = ">=1.7.2,<2"
hdf5 = ">=1.14.4,<2"
h5netcdf = ">=1.5.0,<2"

[tool.pixi.feature.icechunk-dev.dependencies]
rust = "*"

[tool.pixi.feature.minimum-versions.dependencies]
xarray = "==2025.6.0"
numpy = "==2.1.0"
numcodecs = "==0.15.1"
zarr = "==3.1.0"
obstore = "==0.7.0"
icechunk = "==1.1.2"

# Define commands to run within the test environments
[tool.pixi.feature.test.tasks]
run-mypy = { cmd = "mypy virtualizarr" }
# Using '--dist loadscope' (rather than default of '--dist load' when '-n auto'
# is used), reduces test hangs that appear to be macOS-related.
run-tests = { cmd = "pytest -n auto --dist loadscope --run-network-tests --verbose --durations=10" }
run-tests-including-slow = { cmd = "pytest -n auto --dist loadscope --run-network-tests --run-slow-tests --verbose --durations=10" }
run-tests-no-network = { cmd = "pytest -n auto --dist loadscope --verbose" }
run-tests-cov = { cmd = "pytest -n auto --dist loadscope --run-network-tests --verbose --cov=virtualizarr --cov=term-missing" }
run-tests-xml-cov = { cmd = "pytest -n auto --dist loadscope --run-network-tests --verbose --cov=virtualizarr --cov-report=xml" }
run-tests-html-cov = { cmd = "pytest -n auto --dist loadscope --run-network-tests --verbose --cov=virtualizarr --cov-report=html" }
run-flaky-tests = { cmd = "pytest -m flaky --run-flaky --run-network-tests --verbose" }

# Define which features and groups to include in different pixi (similar to conda) environments)
[tool.pixi.environments]
min-deps = ["dev", "test", "hdf", "hdf5-lib"] # VirtualiZarr/conftest.py using h5py, so the minimum set of dependencies for testing still includes hdf libs
# Inherit from min-deps to get all the test commands, along with optional dependencies
test = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "zarr", "py314"]
test-py311 = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "tiff", "zarr","py311"] # test against python 3.11
test-py312 = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "tiff", "zarr", "py312"] # test against python 3.13
test-py313 = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "tiff", "zarr", "py313"] # test against python 3.13
test-py314 = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "zarr", "py314"] # test against python 3.14
minio = ["dev", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "hdf5-lib", "py314", "zarr", "minio"]
minimum-versions = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "tiff", "zarr","minimum-versions", "py311"]
upstream = ["dev", "test", "hdf", "hdf5-lib", "netcdf3", "upstream", "icechunk-dev", "zarr", "py313"]
all = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "tiff", "zarr", "all_parsers", "all_writers", "py313"]
docs = ["docs", "dev", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "tiff","zarr", "py313"]

[tool.pixi.dependencies]
pytest = "*"

# Define commands to run within the docs environment
[tool.pixi.feature.docs.tasks]
serve-docs = { cmd = "mkdocs serve" }
build-docs = { cmd = "mkdocs build" }
check-docs = { cmd = "mkdocs build --strict" }
readthedocs = { cmd = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r site $READTHEDOCS_OUTPUT/html" }
# Define commands to run within the docs environment
[tool.pixi.feature.minio.tasks]
run-tests = { cmd = "pytest virtualizarr/tests/test_manifests/test_store.py virtualizarr/tests/test_parsers/test_hdf/test_hdf_manifest_store.py virtualizarr/tests/test_parsers/test_zarr.py --run-minio-tests --run-network-tests --verbose" }
run-tests-xml-cov = { cmd = "pytest virtualizarr/tests/test_manifests/test_store.py virtualizarr/tests/test_parsers/test_hdf/test_hdf_manifest_store.py virtualizarr/tests/test_parsers/test_zarr.py --run-minio-tests --run-network-tests --verbose --cov-report=xml" }

[tool.setuptools_scm]
fallback_version = "9999"

[tool.setuptools.packages.find]
exclude = ["docs", "tests", "tests.*", "docs.*"]

[tool.setuptools.package-data]
datatree = ["py.typed"]

[tool.mypy]
files = "virtualizarr/**/*.py"
show_error_codes = true

[[tool.mypy.overrides]]
module = [
    "docker",
    "fsspec.*",
    "s3fs.*",
    "h5py",
    "kerchunk.*",
    "minio",
    "numcodecs.*",
    "ujson",
    "zarr",
    "requests",
]
ignore_missing_imports = true

[tool.ruff]
# Same as Black.
line-length = 88
indent-width = 4
target-version = "py310"

exclude = [
    "docs",
    ".eggs"
]

[tool.ruff.lint]
# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.
# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
# McCabe complexity (`C901`) by default.
select = ["E4", "E7", "E9", "F", "I"]
per-file-ignores = {}

# E402: module level import not at top of file
# E731: do not assign a lambda expression, use a def
# W503: line break before binary operator - not implimeted by ruff due to conflict with PEP8.

ignore = ["E402", "E731"]

# Allow fix for all enabled rules (when `--fix`) is provided.
fixable = ["ALL"]
unfixable = []

[tool.ruff.format]
# Like Black, use double quotes for strings.
quote-style = "double"
# Indent with spaces, rather than tabs.
indent-style = "space"
# Respect magic trailing commas.
skip-magic-trailing-comma = false
# Automatically detect the appropriate line ending.
line-ending = "auto"

[tool.ruff.lint.isort]
known-first-party = ["virtualizarr"]

[tool.coverage.run]
omit = ["conftest.py", "virtualizarr/tests/*"]

[tool.coverage.report]
exclude_lines = [
    "pragma: no cover",
    "if TYPE_CHECKING:",
    "pragma: ${PY_MAJOR_VERSION} no cover",
    '.*\.\.\.' # Ignore "..." lines
]

[tool.pytest.ini_options]
addopts = ["--doctest-modules", "--ignore=examples"]
# See https://pytest-asyncio.readthedocs.io/en/latest/concepts.html#asyncio-event-loops
# Explicitly set asyncio_default_fixture_loop_scope to eliminate the following warning:
#
#    PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope"
#    is unset. The event loop scope for asynchronous fixtures will default to the fixture
#    caching scope. Future versions of pytest-asyncio will default the loop scope for
#    asynchronous fixtures to function scope. Set the default fixture loop scope
#    explicitly in order to avoid unexpected behavior in the future. Valid fixture loop
#    scopes are: "function", "class", "module", "package", "session"
#
timeout = 300
asyncio_default_fixture_loop_scope = "session"
markers = [
    "flaky: flaky tests (skipped by default, run with '--run-flaky')",
    "network: marks test requiring internet (select with '--run-network-tests')",
    "slow: marks test as slow (select with '--run-slow-tests')",
    "minio: marks test requiring docker and minio (select with '--run-minio-tests')",
]
filterwarnings = [
    "ignore:Numcodecs codecs are not in the Zarr version 3 specification*:UserWarning:numcodecs"
]

[tool.numpydoc_validation]
# See https://numpydoc.readthedocs.io/en/latest/validation.html#built-in-validation-checks for list of checks
checks = [
    "GL06",
    "GL07",
    # Currently broken; see https://github.com/numpy/numpydoc/issues/573
    # "GL09",
    "GL10",
    "SS02",
    "SS04",
    "PR02",
    "PR03",
    "PR05",
    "PR06",
]