From f17b5cd08dbbbb8ec718af7d83eb0fd45e0aeffd Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 27 Oct 2026 18:18:12 +0100 Subject: [PATCH 1/8] feat(zarr-metadata): add Hypothesis strategies for metadata fields Add `zarr_metadata.testing.strategies`, behind a new `testing` extra. It ships Hypothesis strategies that draw valid Zarr v3 metadata fields, one for every codec, data type, chunk grid and chunk key encoding the package defines that holds no other field. - Each kind's strategy takes what its configurations depend on: a data type's fill values take its configuration, a chunk grid takes the array shape, and a codec takes the `Chunk` it is handed. - `fields` draws every envelope spelling the spec allows, or only the canonical one. - Strategies are a mapping keyed by definition, so an extension author can add strategies for their own definitions. The tests read every draw back in `CORE_AND_EXTENSIONS` and require no problems. They also check: - that every definition in scope has a strategy, apart from the three that hold nested fields; - reach, with targets derived from each configuration's TypedDict: each optional key present and absent, each `Literal` value, and a non-canonical spelling where one exists; - fill-value spellings and envelope forms. Strategies follow the spec where the package is looser. Core data types are drawn only as bare strings, and regular-grid chunk lengths are never zero ("Chunk sizes must be greater than zero"), although the package's rules accept both today. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/changes/0000.feature.md | 1 + packages/zarr-metadata/docs/api/testing.md | 8 + packages/zarr-metadata/mkdocs.yml | 1 + packages/zarr-metadata/pyproject.toml | 5 + .../src/zarr_metadata/testing/__init__.py | 6 + .../src/zarr_metadata/testing/_chunk_grid.py | 107 +++++ .../testing/_chunk_key_encoding.py | 40 ++ .../src/zarr_metadata/testing/_codec.py | 136 +++++++ .../src/zarr_metadata/testing/_data_type.py | 177 +++++++++ .../src/zarr_metadata/testing/_strategy.py | 132 +++++++ .../src/zarr_metadata/testing/strategies.py | 273 +++++++++++++ .../src/zarr_metadata/v3/_definition.py | 4 +- .../zarr-metadata/tests/test_public_api.py | 5 + .../zarr-metadata/tests/testing/__init__.py | 0 .../tests/testing/test_strategies.py | 370 ++++++++++++++++++ packages/zarr-metadata/uv.lock | 13 +- 16 files changed, 1274 insertions(+), 4 deletions(-) create mode 100644 packages/zarr-metadata/changes/0000.feature.md create mode 100644 packages/zarr-metadata/docs/api/testing.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_codec.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/strategies.py create mode 100644 packages/zarr-metadata/tests/testing/__init__.py create mode 100644 packages/zarr-metadata/tests/testing/test_strategies.py diff --git a/packages/zarr-metadata/changes/0000.feature.md b/packages/zarr-metadata/changes/0000.feature.md new file mode 100644 index 0000000000..bd1d87ab40 --- /dev/null +++ b/packages/zarr-metadata/changes/0000.feature.md @@ -0,0 +1 @@ +Added `zarr_metadata.testing.strategies`, Hypothesis strategies that draw valid Zarr v3 metadata fields, behind a new `testing` extra (`pip install zarr-metadata[testing]`). There is one strategy for every codec, data type, chunk grid and chunk key encoding the package defines that holds no other field. `fields` draws every envelope spelling the spec allows, or only the canonical one with `spellings="canonical"`. Codecs can be drawn for the `Chunk` they are handed, chunk grids for an array shape, and fill values for a data type. The strategies are a mapping keyed by definition, so an extension author can add strategies for their own definitions. diff --git a/packages/zarr-metadata/docs/api/testing.md b/packages/zarr-metadata/docs/api/testing.md new file mode 100644 index 0000000000..a5d6145258 --- /dev/null +++ b/packages/zarr-metadata/docs/api/testing.md @@ -0,0 +1,8 @@ +--- +title: testing.strategies +--- + +::: zarr_metadata.testing.strategies + options: + inherited_members: false + show_if_no_docstring: false diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 7708513d24..d87d91ed97 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -16,6 +16,7 @@ nav: - api/index.md - ' zarr_metadata.model': api/model.md - ' zarr_metadata.pydantic': api/pydantic.md + - ' zarr_metadata.testing.strategies': api/testing.md - ' zarr_metadata.typed_json': api/typed_json.md - ' zarr_metadata.v2': api/v2.md - ' zarr_metadata.v3': diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 31d548bf2a..c242eb547f 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -39,6 +39,11 @@ dependencies = [ "typing_extensions>=4.16", ] +[project.optional-dependencies] +# `zarr_metadata.testing.strategies`. 6.85: first release with +# `characters(exclude_categories=...)`. +testing = ["hypothesis>=6.85"] + [project.urls] Homepage = "https://github.com/zarr-developers/zarr-python" Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py b/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py new file mode 100644 index 0000000000..271a4dfe51 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py @@ -0,0 +1,6 @@ +"""Testing utilities for code that reads or writes Zarr metadata. + +`zarr_metadata.testing.strategies` holds Hypothesis strategies that draw +valid metadata. It needs the `testing` extra: +`pip install zarr-metadata[testing]`. +""" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py new file mode 100644 index 0000000000..bbefe58fe3 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py @@ -0,0 +1,107 @@ +"""Strategies for each chunk grid this package defines. + +Over an array of a known shape, a grid is drawn to fit it: a chunk +length for each dimension, never zero ("Chunk sizes must be greater +than zero", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L40), +and rectilinear edges that cover each dimension. Over no shape, what is +drawn is what the grid's rules accept. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import ChunkGridStrategy +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import ChunkGridDefinition + +_MAX_RANK: Final = 5 +"""The most dimensions a grid is drawn with, over no shape.""" + +_MAX_LENGTH: Final = 16 +"""The longest chunk drawn along a dimension whose length is unknown.""" + +_MAX_RUNS: Final = 5 +"""The most runs of equal lengths a rectilinear dimension is drawn with.""" + +_MAX_COUNT: Final = 4 +"""The most chunks a run is drawn with.""" + + +def _length(extent: int | None) -> st.SearchStrategy[int]: + """A chunk length along a dimension of `extent`: at least 1, and up to twice the extent when it is known and not 0.""" + if extent is None or extent == 0: + return st.integers(1, _MAX_LENGTH) + return st.integers(1, 2 * extent) + + +def _regular(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """A chunk length for each dimension.""" + if shape is None: + lengths = st.lists(_length(None), max_size=_MAX_RANK).map(tuple) + else: + lengths = st.tuples(*(_length(extent) for extent in shape)) + return lengths.map(lambda chunk_shape: {"chunk_shape": chunk_shape}) + + +def _spelled_run(draw: st.DrawFn, size: int, count: int) -> list[JSONValue]: + """`count` chunks of `size`: one `[size, count]` pair, or the size written once for each.""" + if draw(st.booleans()): + return [(size, count)] + return [size] * count + + +@st.composite +def _edges(draw: st.DrawFn, extent: int | None) -> JSONValue: + """Edge lengths along a dimension, covering `extent`, in runs spelled as pairs or written out. + + Runs are drawn apart, so two of one size can sit side by side and a + pair can count one chunk: spellings the canonical form merges. A + dimension of length zero is covered by no edges at all. + """ + min_runs = 0 if extent == 0 else 1 + runs = draw( + st.lists( + st.tuples(st.integers(1, _MAX_LENGTH), st.integers(1, _MAX_COUNT)), + min_size=min_runs, + max_size=_MAX_RUNS, + ) + ) + covered = sum(size * count for size, count in runs) + if extent is not None and covered < extent: + runs.append((extent - covered, 1)) + spelled: list[JSONValue] = [] + for size, count in runs: + spelled.extend(_spelled_run(draw, size, count)) + return tuple(spelled) + + +def _rectilinear_dimension(extent: int | None) -> st.SearchStrategy[JSONValue]: + """A bare length, which repeats until it covers the dimension, or edges that cover it.""" + return _length(extent) | _edges(extent) + + +def _rectilinear(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """Inline chunk shapes, one entry for each dimension.""" + if shape is None: + dimensions = st.lists(_rectilinear_dimension(None), max_size=_MAX_RANK).map(tuple) + else: + dimensions = st.tuples(*(_rectilinear_dimension(extent) for extent in shape)) + return dimensions.map(lambda chunk_shapes: {"kind": "inline", "chunk_shapes": chunk_shapes}) + + +CHUNK_GRID_STRATEGIES: Final[Mapping[ChunkGridDefinition[Any], ChunkGridStrategy]] = { + REGULAR_CHUNK_GRID: ChunkGridStrategy(configurations=_regular), + RECTILINEAR_CHUNK_GRID: ChunkGridStrategy(configurations=_rectilinear), +} +"""A strategy for each chunk grid.""" + +__all__ = ["CHUNK_GRID_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py new file mode 100644 index 0000000000..9c525118c9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py @@ -0,0 +1,40 @@ +"""Strategies for each chunk key encoding this package defines.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import ChunkKeyEncodingStrategy +from zarr_metadata.v3.chunk_key_encoding.default import ( + DEFAULT_CHUNK_KEY_ENCODING, + DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, +) +from zarr_metadata.v3.chunk_key_encoding.v2 import ( + V2_CHUNK_KEY_ENCODING, + V2_CHUNK_KEY_ENCODING_SEPARATOR, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition + +CHUNK_KEY_ENCODING_STRATEGIES: Final[ + Mapping[ChunkKeyEncodingDefinition[Any], ChunkKeyEncodingStrategy] +] = { + DEFAULT_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( + configurations=st.fixed_dictionaries( + {}, optional={"separator": st.sampled_from(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)} + ) + ), + V2_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( + configurations=st.fixed_dictionaries( + {}, optional={"separator": st.sampled_from(V2_CHUNK_KEY_ENCODING_SEPARATOR)} + ) + ), +} +"""A strategy for each chunk key encoding: a `separator`, or none, which means the encoding's default.""" + +__all__ = ["CHUNK_KEY_ENCODING_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py new file mode 100644 index 0000000000..f622c35467 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -0,0 +1,136 @@ +"""Strategies for each codec this package defines that holds no other field. + +A codec handed an array draws what fits the chunk: a `bytes` codec an +`endian` for a type whose values take several bytes, a `transpose` an +`order` with an entry for each axis. Handed `Chunk()`, nothing is known +of the chunk, and what is drawn is what the codec's rules accept. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._data_type import FLOAT_WIDTHS, INTEGER_RANGES, finite_floats +from zarr_metadata.testing._strategy import CodecStrategy +from zarr_metadata.v3._definition import Chunk, storage_of +from zarr_metadata.v3.codec.blosc import BLOSC_CNAME, BLOSC_CODEC, BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE +from zarr_metadata.v3.codec.bytes import BYTES_CODEC, ENDIANNESS +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC +from zarr_metadata.v3.codec.gzip import GZIP_CODEC +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import CodecDefinition + +_MAX_RANK: Final = 5 +"""The most axes an `order` is drawn with, when the chunk does not say.""" + +_BLOSC_MAX_TYPESIZE: Final = 255 +"""The widest element c-blosc takes: `BLOSC_MAX_TYPESIZE`.""" + +_BLOSC_MAX_BLOCKSIZE: Final = 2**31 - 1 +"""The largest block c-blosc takes, a signed 32-bit size; 0 lets it choose.""" + +_ENDIAN: Final = st.sampled_from(ENDIANNESS) + + +def _bytes(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: + """An `endian`, required for a data type whose values take several bytes; None for one whose values vary in size.""" + storage = None if chunk.data_type is None else storage_of(chunk.data_type) + if storage == "variable_length": + return None + if storage == "multi_byte": + return st.fixed_dictionaries({"endian": _ENDIAN}) + return st.fixed_dictionaries({}, optional={"endian": _ENDIAN}) + + +@st.composite +def _blosc(draw: st.DrawFn) -> Mapping[str, JSONValue]: + """A compressor, a level, a shuffle and a block size; and a `typesize`, which only `noshuffle` may leave out. + + Under `noshuffle` the spec says `typesize` "is ignored", so it is + drawn there too, a spelling that is not the simplest. + """ + shuffle = draw(st.sampled_from(BLOSC_SHUFFLE)) + typesize = st.integers(1, _BLOSC_MAX_TYPESIZE) + required: dict[str, st.SearchStrategy[JSONValue]] = { + "cname": st.sampled_from(BLOSC_CNAME), + "clevel": st.integers(0, 9), + "shuffle": st.just(shuffle), + "blocksize": st.integers(0, _BLOSC_MAX_BLOCKSIZE), + } + if shuffle == BLOSC_NO_SHUFFLE: + return draw(st.fixed_dictionaries(required, optional={"typesize": typesize})) + return draw(st.fixed_dictionaries({**required, "typesize": typesize})) + + +def _order(rank: int) -> st.SearchStrategy[Mapping[str, JSONValue]]: + return st.permutations(range(rank)).map(lambda order: {"order": tuple(order)}) + + +def _transpose(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """A permutation of the chunk's axes; of some number of axes, when the chunk does not say.""" + if chunk.rank is not None: + return _order(chunk.rank) + return st.integers(0, _MAX_RANK).flatmap(_order) + + +def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: + """An `offset` and a `scale`, each a finite value of the data type the codec is handed, the scale not zero. + + Each is a fill value of the chunk's data type + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), + drawn finite, and the scale non-zero, so that decoding, which divides + by it, is defined. Handed a data type the codec's registry entry does + not list -- truth values, raw bits, complex numbers, numpy times -- + nothing is drawn: for some the spec refuses it, and for the others + leaves it open. Handed a chunk whose data type is unknown, the values + are small non-negative integers, which every listed type holds. + """ + data_type = chunk.data_type + read = data_type is not None and data_type.resolution == "read" + definition = data_type.definition if data_type is not None and read else None + if definition is None: + offset: st.SearchStrategy[JSONValue] = st.integers(0, 127) + scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) + elif definition in INTEGER_RANGES: + low, high = INTEGER_RANGES[definition] + offset = st.integers(low, high) + scale = st.integers(low, high).filter(lambda value: value != 0) + elif definition in FLOAT_WIDTHS: + offset = finite_floats(FLOAT_WIDTHS[definition]) + scale = finite_floats(FLOAT_WIDTHS[definition]).filter(lambda value: value != 0) + else: + return None + return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) + + +def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> CodecStrategy: + """A codec whose configurations do not depend on the chunk it is handed.""" + return CodecStrategy(configurations=lambda _chunk: configurations) + + +CODEC_STRATEGIES: Final[Mapping[CodecDefinition[Any], CodecStrategy]] = { + BLOSC_CODEC: _unchunked(_blosc()), + BYTES_CODEC: CodecStrategy(configurations=_bytes), + CRC32C_CODEC: _unchunked(st.just({})), + GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), + SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), + TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), + ZSTD_CODEC: _unchunked( + st.fixed_dictionaries( + {"level": st.integers(ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL)}, + optional={"checksum": st.booleans()}, + ) + ), +} +"""A strategy for each codec that holds no other field.""" + +__all__ = ["CODEC_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py new file mode 100644 index 0000000000..a18faceee2 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -0,0 +1,177 @@ +"""Strategies for each data type this package defines that holds no other field. + +A fill value is drawn in every spelling the data type's spec gives it: +a float's as a number, a named non-finite value or a hex string of its +bits, a `bytes` value's as an array of byte values or base64. +""" + +from __future__ import annotations + +import base64 +from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_args + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import DataTypeStrategy +from zarr_metadata.v3._definition import EmptyConfiguration +from zarr_metadata.v3.data_type._float import FloatSpecialFillValue +from zarr_metadata.v3.data_type._numpy_time import NUMPY_TIME_MAX_SCALE_FACTOR +from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE +from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE +from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE +from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE +from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import ( + NUMPY_TIME_UNIT, + NUMPY_TIMEDELTA64_DATA_TYPE, +) +from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration +from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE +from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE +from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE +from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE +from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import DataTypeDefinition + +FloatWidth = Literal[16, 32, 64] + +INTEGER_RANGES: Final[Mapping[DataTypeDefinition[Any], tuple[int, int]]] = { + INT8_DATA_TYPE: (-(2**7), 2**7 - 1), + INT16_DATA_TYPE: (-(2**15), 2**15 - 1), + INT32_DATA_TYPE: (-(2**31), 2**31 - 1), + INT64_DATA_TYPE: (-(2**63), 2**63 - 1), + UINT8_DATA_TYPE: (0, 2**8 - 1), + UINT16_DATA_TYPE: (0, 2**16 - 1), + UINT32_DATA_TYPE: (0, 2**32 - 1), + UINT64_DATA_TYPE: (0, 2**64 - 1), +} +"""The values each integer data type holds.""" + +FLOAT_WIDTHS: Final[Mapping[DataTypeDefinition[Any], FloatWidth]] = { + FLOAT16_DATA_TYPE: 16, + FLOAT32_DATA_TYPE: 32, + FLOAT64_DATA_TYPE: 64, +} +"""The width in bits of each floating-point data type.""" + +_EMPTY: Final = st.just(EmptyConfiguration()) + +_MAX_RAW_BYTES: Final = 32 +"""The widest raw bits drawn, in bytes: `r256`.""" + +_MAX_BYTES: Final = 16 +"""The longest `bytes` fill value drawn.""" + + +def finite_floats(width: FloatWidth) -> st.SearchStrategy[float]: + """Finite numbers a float of `width` bits holds exactly.""" + return st.floats(width=width, allow_nan=False, allow_infinity=False) + + +def _hex_spellings(digits: str) -> st.SearchStrategy[str]: + """`0x` and the digits, the letters in either case.""" + return st.sampled_from((f"0x{digits}", f"0x{digits.upper()}")) + + +def _float_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A number, an integer the type holds exactly, a named non-finite value, or the bits in hex. + + A float's fill value is any of these + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79). + Numbers are drawn from those the type holds, so none is rounded: a + `float16` takes `1e300` by the letter of the spec, and a reader is + free to make of it what it will. + """ + # Every integer of magnitude up to 2**11 is a float16, the narrowest type. + numbers = finite_floats(width) | st.integers(-(2**11), 2**11) + named = st.sampled_from(get_args(FloatSpecialFillValue)) + bits = st.integers(0, 2**width - 1).flatmap( + lambda value: _hex_spellings(f"{value:0{width // 4}x}") + ) + return st.one_of(numbers, named, bits) + + +def _fill_values_of(values: st.SearchStrategy[JSONValue]) -> DataTypeStrategy: + """A data type with nothing to configure, whose fill values are `values`.""" + return DataTypeStrategy(configurations=_EMPTY, fill_values=lambda _configuration: values) + + +def _byte_values(size: int) -> st.SearchStrategy[JSONValue]: + """`size` byte values, each an integer in `[0, 255]`, as a JSON array.""" + return st.lists(st.integers(0, 255), min_size=size, max_size=size).map( + lambda values: cast("JSONValue", tuple(values)) + ) + + +def _raw_bytes_fill_values(configuration: Mapping[str, JSONValue]) -> st.SearchStrategy[JSONValue]: + """One byte value for each 8 bits of the size.""" + return _byte_values(cast("RawBytesConfiguration", configuration)["bits"] // 8) + + +def _base64(value: bytes) -> JSONValue: + return base64.b64encode(value).decode("ascii") + + +def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A pair of float fill values of `width` bits, real then imaginary.""" + component = _float_fill_values(width) + return st.tuples(component, component).map(lambda pair: cast("JSONValue", pair)) + + +_BYTES_FILL_VALUES: Final = st.integers(0, _MAX_BYTES).flatmap(_byte_values) | st.binary( + max_size=_MAX_BYTES +).map(_base64) +"""A `bytes` fill value: byte values, or the bytes in standard-alphabet base64.""" + +_TEXT: Final = st.text(st.characters(exclude_categories=("Cs",)), max_size=16) +"""Text a JSON document can hold: no lone surrogate, which is no Unicode scalar value.""" + +_NUMPY_TIME: Final = DataTypeStrategy( + configurations=st.fixed_dictionaries( + { + "unit": st.sampled_from(NUMPY_TIME_UNIT), + "scale_factor": st.integers(1, NUMPY_TIME_MAX_SCALE_FACTOR), + } + ), + fill_values=lambda _configuration: st.integers(-(2**63), 2**63 - 1) | st.just("NaT"), +) +"""Both numpy time types: a unit, and how many of it one tick is; a signed 64-bit count of ticks, or `"NaT"`.""" + +DATA_TYPE_STRATEGIES: Final[Mapping[DataTypeDefinition[Any], DataTypeStrategy]] = { + BOOL_DATA_TYPE: _fill_values_of(st.booleans()), + **{ + definition: _fill_values_of(st.integers(low, high)) + for definition, (low, high) in INTEGER_RANGES.items() + }, + **{ + definition: _fill_values_of(_float_fill_values(width)) + for definition, width in FLOAT_WIDTHS.items() + }, + COMPLEX64_DATA_TYPE: _fill_values_of(_complex_fill_values(32)), + COMPLEX128_DATA_TYPE: _fill_values_of(_complex_fill_values(64)), + RAW_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=st.integers(1, _MAX_RAW_BYTES).map( + lambda size: RawBytesConfiguration(bits=8 * size) + ), + fill_values=_raw_bytes_fill_values, + ), + BYTES_DATA_TYPE: _fill_values_of(_BYTES_FILL_VALUES), + STRING_DATA_TYPE: _fill_values_of(_TEXT), + NUMPY_DATETIME64_DATA_TYPE: _NUMPY_TIME, + NUMPY_TIMEDELTA64_DATA_TYPE: _NUMPY_TIME, +} +"""A strategy for each data type that holds no other field.""" + +__all__ = ["DATA_TYPE_STRATEGIES", "FLOAT_WIDTHS", "INTEGER_RANGES", "finite_floats"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py new file mode 100644 index 0000000000..0336bcb99e --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py @@ -0,0 +1,132 @@ +"""What draws one definition's configurations: a record for each kind of definition. + +Each record holds what its kind's configurations depend on, the way +each kind of definition does: a data type's fill values depend on its +configuration, a chunk grid's on the shape of the array, a codec's on +the chunk it is handed. A chunk key encoding's depend on nothing. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, cast + +from zarr_metadata.v3._definition import ( + Chunk, + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + DataTypeDefinition, + Definition, +) + +if TYPE_CHECKING: + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + +Configurations: TypeAlias = "SearchStrategy[Mapping[str, JSONValue]]" +"""Configurations of one definition, each the mapping every configuration is.""" + +NOTHING_KNOWN: Final = Chunk() +"""A chunk nothing is known of, which every codec takes.""" + + +class DefinitionStrategy(ABC): + """How to draw the configurations of one kind of definition.""" + + __slots__ = () + + kind: ClassVar[type[Definition[Any]]] + """The kind of definition whose configurations it draws.""" + + @abstractmethod + def unconstrained(self) -> Configurations: + """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DataTypeStrategy(DefinitionStrategy): + """How to draw a data type's configurations, and the fill values an array of one takes.""" + + kind: ClassVar[type[Definition[Any]]] = DataTypeDefinition + + configurations: Configurations + """Configurations its definition accepts.""" + fill_values: Callable[[Mapping[str, JSONValue]], SearchStrategy[JSONValue]] + """Fill values of the data type a configuration describes.""" + + def unconstrained(self) -> Configurations: + return self.configurations + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkGridStrategy(DefinitionStrategy): + """How to draw a chunk grid's configurations over an array of a shape.""" + + kind: ClassVar[type[Definition[Any]]] = ChunkGridDefinition + + configurations: Callable[[tuple[int, ...] | None], Configurations] + """Configurations of grids that fit an array of the shape. + + Handed None, no shape is known, and what is drawn is what the grid's + rules accept, whatever shape it is put over. + """ + + def unconstrained(self) -> Configurations: + return self.configurations(None) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkKeyEncodingStrategy(DefinitionStrategy): + """How to draw a chunk key encoding's configurations.""" + + kind: ClassVar[type[Definition[Any]]] = ChunkKeyEncodingDefinition + + configurations: Configurations + """Configurations its definition accepts.""" + + def unconstrained(self) -> Configurations: + return self.configurations + + +@dataclass(frozen=True, kw_only=True, slots=True) +class CodecStrategy(DefinitionStrategy): + """How to draw a codec's configurations, handed a chunk. + + Every codec takes `Chunk()`, a chunk nothing is known of, so a + strategy that draws nothing for it is refused when it is built. + """ + + kind: ClassVar[type[Definition[Any]]] = CodecDefinition + + configurations: Callable[[Chunk], Configurations | None] + """Configurations of the codec handed the chunk; None when no configuration takes it. + + A `bytes` codec handed a `string` chunk is None, since the spec gives + that data type a codec of its own. + """ + + def __post_init__(self) -> None: + if self.configurations(NOTHING_KNOWN) is None: + msg = "a codec strategy draws configurations for Chunk(), a chunk nothing is known of; got None" + raise ValueError(msg) + + def unconstrained(self) -> Configurations: + return cast("Configurations", self.configurations(NOTHING_KNOWN)) + + +Strategies: TypeAlias = Mapping[Definition[Any], DefinitionStrategy] +"""How to draw each definition's configurations, by the definition.""" + +__all__ = [ + "ChunkGridStrategy", + "ChunkKeyEncodingStrategy", + "CodecStrategy", + "Configurations", + "DataTypeStrategy", + "DefinitionStrategy", + "Strategies", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py new file mode 100644 index 0000000000..991be8fc3b --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -0,0 +1,273 @@ +"""Hypothesis strategies for valid Zarr v3 metadata fields. + +Every strategy here draws metadata its definition accepts, and that the +spec allows: a reader, writer or converter can test itself against the +whole space of a codec's, data type's, chunk grid's or chunk key +encoding's configurations instead of the handful its authors thought to +write down. + +```python +from hypothesis import given +from zarr_metadata.testing.strategies import fields +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC + +@given(fields(BLOSC_CODEC)) +def test_my_reader_reads_every_blosc_codec(field): + my_reader.read_codec(field) +``` + +What a configuration depends on, each kind's function takes: +`fill_values` a data type's configuration, `chunk_grid_configurations` +the shape of the array, `codec_configurations` the `Chunk` the codec is +handed. Left out, nothing is known of it, and what is drawn is what the +definition's rules accept. `fields` wraps configurations in the +envelopes a document writes them in. + +How each definition's configurations are drawn is a value, +`Strategies`, keyed by the definition: `DEFAULT_STRATEGIES` holds one +for each definition this package defines that holds no other field. An +extension author adds their own with +`DEFAULT_STRATEGIES | {MY_CODEC: CodecStrategy(...)}`. + +Configurations are drawn at the data level the TypedDicts declare, JSON +arrays as tuples; `json.dumps` takes them as they are. + +Requires the `testing` extra (`pip install zarr-metadata[testing]`). +""" + +from __future__ import annotations + +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast + +from hypothesis import strategies as st + +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.testing._chunk_grid import CHUNK_GRID_STRATEGIES +from zarr_metadata.testing._chunk_key_encoding import CHUNK_KEY_ENCODING_STRATEGIES +from zarr_metadata.testing._codec import CODEC_STRATEGIES +from zarr_metadata.testing._data_type import DATA_TYPE_STRATEGIES +from zarr_metadata.testing._strategy import ( + NOTHING_KNOWN, + ChunkGridStrategy, + ChunkKeyEncodingStrategy, + CodecStrategy, + Configurations, + DataTypeStrategy, + DefinitionStrategy, + Strategies, +) +from zarr_metadata.v3._definition import ( + Chunk, + DataTypeDefinition, + Definition, + as_kind, + canonicalize, + carrying_name, + kind_of, +) +from zarr_metadata.v3._registry import CORE, Context + +if TYPE_CHECKING: + from collections.abc import Mapping + + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import ( + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + ) + from zarr_metadata.v3.definition import ZarrV3MetadataFieldJSON + +C = TypeVar("C") + +Spellings = Literal["all", "canonical"] +"""How fields are written: every spelling the spec allows, or only the simplest. + +`"all"` draws each envelope the spec gives a field -- a bare name, an +object without a configuration, an empty configuration, a +`must_understand` of `true` -- and configurations that say what a +simpler one says, as a `blosc` `typesize` under `noshuffle` does. +`"canonical"` draws each in the spelling `canonicalize` gives it, which +is what a writer that writes the fewest words writes. +""" + +DEFAULT_STRATEGIES: Final[MappingProxyType[Definition[Any], DefinitionStrategy]] = MappingProxyType( + { + **DATA_TYPE_STRATEGIES, + **CHUNK_GRID_STRATEGIES, + **CHUNK_KEY_ENCODING_STRATEGIES, + **CODEC_STRATEGIES, + } +) +"""A strategy for each definition this package defines that holds no other field.""" + + +def _strategy_for(definition: Definition[Any], strategies: Strategies) -> DefinitionStrategy: + """The strategy `strategies` holds for `definition`, which draws configurations of its kind.""" + found = strategies.get(definition) + if found is None: + msg = f"no strategy for {definition.name!r}; pass one in `strategies`" + raise LookupError(msg) + if not isinstance(definition, found.kind): + msg = ( + f"the strategy for {definition.name!r}, a {type(definition).__name__}, draws " + f"configurations of a {found.kind.__name__}: {found!r}" + ) + raise TypeError(msg) + return found + + +def data_type_configurations( + definition: DataTypeDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[C]: + """Configurations of the data type `definition` defines.""" + found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations) + + +def fill_values( + definition: DataTypeDefinition[C], + configuration: C, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[JSONValue]: + """Fill values of the data type `definition` and `configuration` describe, in every spelling the spec gives them.""" + found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) + return found.fill_values(cast("Mapping[str, JSONValue]", configuration)) + + +def chunk_grid_configurations( + definition: ChunkGridDefinition[C], + shape: tuple[int, ...] | None = None, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[C]: + """Configurations of the chunk grid `definition` defines, that fit an array of `shape`. + + With no shape, what is drawn is what the grid's rules accept. + """ + found = cast("ChunkGridStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations(shape)) + + +def chunk_key_encoding_configurations( + definition: ChunkKeyEncodingDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[C]: + """Configurations of the chunk key encoding `definition` defines.""" + found = cast("ChunkKeyEncodingStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations) + + +def codec_configurations( + definition: CodecDefinition[C], + chunk: Chunk = NOTHING_KNOWN, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[C] | None: + """Configurations of the codec `definition` defines, handed `chunk`; None when none takes it. + + With no chunk, nothing is known of it, and what is drawn is what the + codec's rules accept. + """ + found = cast("CodecStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C] | None", found.configurations(chunk)) + + +def _written_bare(definition: Definition[Any]) -> bool: + """Whether the spec has the field written as its bare name: a data type it defines itself. + + "If the data type is defined in this specification, then the value + must be the data type identifier provided as a string" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L509-L512). + """ + if not isinstance(definition, DataTypeDefinition): + return False + return CORE.claimant(DataTypeDefinition, definition.name) is definition + + +@st.composite +def _any_spelling( + draw: st.DrawFn, definition: Definition[Any], configurations: SearchStrategy[Any] +) -> ZarrV3MetadataFieldJSON: + """A configuration of `definition` in one of the envelopes the spec allows it. + + A configuration with nothing in it is written as the bare name, an + object without one, or an object with an empty one; any object may + say `must_understand` is `true`, which is what absence means. A name + that carries the configuration, as `r16` does, and a data type the + spec defines itself, are written as the bare name. + """ + configuration = cast("Mapping[str, JSONValue]", draw(configurations)) + carried = carrying_name(definition, configuration) + if carried is not None: + return carried + if _written_bare(definition): + return definition.name + empty = len(configuration) == 0 + if empty and draw(st.booleans()): + return definition.name + field: dict[str, JSONValue] = {"name": definition.name} + if not empty or draw(st.booleans()): + field["configuration"] = configuration + if draw(st.booleans()): + field["must_understand"] = True + return cast("ZarrV3MetadataFieldJSON", field) + + +def _canonical( + definition: Definition[Any], field: ZarrV3MetadataFieldJSON +) -> ZarrV3MetadataFieldJSON: + """`field` in the spelling `canonicalize` gives it, read in a scope of its own definition.""" + kind = as_kind(kind_of(definition)) + canonical, problems = canonicalize(field, kind, Context.of(definition)) + if len(problems) != 0: + msg = f"the strategy for {definition.name!r} drew {field!r}, which is refused: {problems!r}" + raise ValueError(msg) + return cast("ZarrV3MetadataFieldJSON", canonical) + + +def fields( + definition: Definition[C], + configurations: SearchStrategy[C] | UNSET = UNSET, + *, + spellings: Spellings = "all", + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[ZarrV3MetadataFieldJSON]: + """Metadata fields of `definition`: its name and a configuration, in an envelope a document writes. + + `configurations` is what to draw the configuration from, by default + the definition's own with nothing known of what they depend on: pass + `codec_configurations(definition, chunk)` for a codec handed a chunk, + once it is known not to be None. `spellings` says whether every + spelling the spec allows is drawn, or only the simplest. + """ + drawn: Configurations | SearchStrategy[C] = ( + _strategy_for(definition, strategies).unconstrained() + if configurations is UNSET + else configurations + ) + spelled = _any_spelling(definition, drawn) + if spellings == "canonical": + return spelled.map(lambda field: _canonical(definition, field)) + return spelled + + +__all__ = [ + "DEFAULT_STRATEGIES", + "ChunkGridStrategy", + "ChunkKeyEncodingStrategy", + "CodecStrategy", + "DataTypeStrategy", + "DefinitionStrategy", + "Spellings", + "Strategies", + "chunk_grid_configurations", + "chunk_key_encoding_configurations", + "codec_configurations", + "data_type_configurations", + "fields", + "fill_values", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index c18a0bd260..c033d9fb4d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -290,7 +290,7 @@ def spelled( return name, None -def _carrying_name( +def carrying_name( definition: Definition[Any], configuration: Mapping[str, JSONValue] ) -> str | None: """The name that carries `configuration` for `definition`: `r16` for `r*` with `{"bits": 16}`; None when its name carries nothing.""" @@ -1148,7 +1148,7 @@ def _canonical_field(definition: Definition[Any], resolved: Resolved[Any]) -> JS f"{list(refused)!r}" ) raise ValueError(msg) - carrying = _carrying_name(definition, simplified) + carrying = carrying_name(definition, simplified) if carrying is not None: return carrying if len(simplified) == 0: diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index cb88193103..9f7b097a1e 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -273,6 +273,9 @@ def test_all_is_grouped_and_unique() -> None: # A configuration member holding a metadata field of that kind: # `CodecField`, read in the scope its field is read in. "Field", + # How to draw one kind of definition's configurations, in + # `zarr_metadata.testing.strategies`: `CodecStrategy`. + "Strategy", ) _EXTENSION_NAME = re.compile(r"^(?:[A-Z][a-z0-9]*)+?(?:" + "|".join(_EXTENSION_ROLES) + r")$") @@ -314,6 +317,8 @@ def test_all_is_grouped_and_unique() -> None: "ScalarMap", "ScalarMapEntry", "ShardingIndexLocation", + "Spellings", + "Strategies", "Struct", "StructField", "TypedDictKeys", diff --git a/packages/zarr-metadata/tests/testing/__init__.py b/packages/zarr-metadata/tests/testing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py new file mode 100644 index 0000000000..1519782457 --- /dev/null +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -0,0 +1,370 @@ +"""The strategies against the definitions they draw for, and against what they ought to reach. + +Every draw is read in `CORE_AND_EXTENSIONS` and must read without a +problem, by the definition it was drawn for: the field alone, a fill +value beside its data type, a chunk grid over the shape it was drawn +for, a codec handed the chunk it was drawn for. A strategy that only +drew valid metadata could draw almost none, so reach is checked too, +with targets worked out from each configuration's TypedDict rather than +from the strategies: every optional key present and absent, every +`Literal` value, and for a definition with a simplest spelling, a +configuration it simplifies. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Literal, cast, get_args, get_origin + +import pytest +from hypothesis import event, find, given, settings +from hypothesis import strategies as st + +from zarr_metadata.testing.strategies import ( + DEFAULT_STRATEGIES, + ChunkKeyEncodingStrategy, + CodecStrategy, + Spellings, + Strategies, + chunk_grid_configurations, + codec_configurations, + data_type_configurations, + fields, + fill_values, +) +from zarr_metadata.typed_json import typeddict_keys +from zarr_metadata.v3._definition import as_kind, kind_of, unchanged +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC +from zarr_metadata.v3.codec.gzip import GZIP_CODEC +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC +from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC +from zarr_metadata.v3.data_type._float import FloatSpecialFillValue +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE +from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE +from zarr_metadata.v3.definition import ( + CORE, + CORE_AND_EXTENSIONS, + Chunk, + ChunkGridDefinition, + CodecDefinition, + DataTypeDefinition, + Definition, + ZarrV3MetadataFieldJSON, + canonicalize, + chunk_grid_lengths, + fill_value_problems, + resolve, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + +_HOLDING_FIELDS = frozenset({SHARDING_INDEXED_CODEC, CAST_VALUE_CODEC, STRUCT_DATA_TYPE}) +"""The definitions whose configurations hold other fields, which have no strategy yet.""" + +_IN_SCOPE = frozenset( + definition for table in CORE_AND_EXTENSIONS.tables.values() for definition in table.values() +) +_CORE_DATA_TYPES = frozenset(CORE.tables[DataTypeDefinition].values()) + +_DEFINITIONS = sorted(DEFAULT_STRATEGIES, key=lambda definition: definition.name) +_DATA_TYPES = [d for d in _DEFINITIONS if isinstance(d, DataTypeDefinition)] +_CHUNK_GRIDS = [d for d in _DEFINITIONS if isinstance(d, ChunkGridDefinition)] +_CODECS = [d for d in _DEFINITIONS if isinstance(d, CodecDefinition)] + +_OPEN_FOR_SCALE_OFFSET = frozenset( + {"complex64", "complex128", "numpy.datetime64", "numpy.timedelta64"} +) +"""Data types `scale_offset` is drawn for none of, though its rules refuse none: its registry entry leaves them open.""" + +_SHAPES = st.lists(st.integers(0, 12), max_size=4).map(tuple) + + +def _id(value: object) -> str: + if isinstance(value, Definition): + return f"{type(value).__name__}:{value.name}" + return str(value) + + +def _read(definition: Definition[Any], field: object) -> Any: # noqa: ANN401 + """`field` read in `CORE_AND_EXTENSIONS`, asserted to read without a problem by `definition`.""" + resolved, problems = resolve(field, as_kind(kind_of(definition)), CORE_AND_EXTENSIONS) + assert problems == () + assert resolved.resolution == "read" + assert resolved.definition is definition + return resolved + + +def test_every_definition_has_a_strategy() -> None: + """Each definition in scope has a strategy, but those that hold other fields; and none is out of scope.""" + assert _IN_SCOPE - frozenset(DEFAULT_STRATEGIES) == _HOLDING_FIELDS + assert frozenset(DEFAULT_STRATEGIES) <= _IN_SCOPE + + +@pytest.mark.parametrize("spellings", get_args(Spellings)) +@pytest.mark.parametrize("definition", _DEFINITIONS, ids=_id) +@given(data=st.data()) +def test_fields_read( + definition: Definition[Any], spellings: Spellings, data: st.DataObject +) -> None: + """A drawn field reads by its definition, through JSON too; a canonical one is its own simplest spelling.""" + field = data.draw(fields(definition, spellings=spellings)) + _read(definition, field) + _read(definition, json.loads(json.dumps(field))) + if definition in _CORE_DATA_TYPES: + assert isinstance(field, str) + if spellings == "canonical": + canonical, _ = canonicalize(field, as_kind(kind_of(definition)), CORE_AND_EXTENSIONS) + assert canonical == field + + +@pytest.mark.parametrize("definition", _DATA_TYPES, ids=_id) +@given(data=st.data()) +def test_fill_values_fit_their_data_type( + definition: DataTypeDefinition[Any], data: st.DataObject +) -> None: + configuration = data.draw(data_type_configurations(definition)) + data_type = _read(definition, data.draw(fields(definition, st.just(configuration)))) + value = data.draw(fill_values(definition, configuration)) + assert fill_value_problems(data_type, value) == () + assert fill_value_problems(data_type, json.loads(json.dumps(value))) == () + + +@pytest.mark.parametrize("definition", _CHUNK_GRIDS, ids=_id) +@given(data=st.data(), shape=_SHAPES) +def test_chunk_grids_fit_their_shape( + definition: ChunkGridDefinition[Any], data: st.DataObject, shape: tuple[int, ...] +) -> None: + configurations = chunk_grid_configurations(definition, shape) + chunk_grid = _read(definition, data.draw(fields(definition, configurations))) + lengths, problems = chunk_grid_lengths(chunk_grid, shape) + assert problems == () + # "Chunk sizes must be greater than zero", which the regular grid's rules do not yet refuse. + assert all(0 not in axis for axis in lengths if axis is not None) + + +@st.composite +def _chunks(draw: st.DrawFn) -> Chunk: + """A chunk of some lengths, of a data type drawn from the strategies.""" + definition = draw(st.sampled_from(_DATA_TYPES)) + data_type = _read(definition, draw(fields(definition))) + lengths = draw(st.lists(st.integers(1, 8), max_size=4)) + return Chunk(tuple(frozenset({length}) for length in lengths), data_type) + + +@pytest.mark.parametrize("definition", _CODECS, ids=_id) +@given(data=st.data(), chunk=_chunks()) +def test_codecs_take_their_chunk( + definition: CodecDefinition[Any], data: st.DataObject, chunk: Chunk +) -> None: + """A codec drawn for a chunk takes it; one drawn for none is one the spec does not give it.""" + assert chunk.data_type is not None + assert chunk.data_type.definition is not None + data_type_name = chunk.data_type.definition.name + configurations = codec_configurations(definition, chunk) + if configurations is None: + event(f"takes no {data_type_name}") + if definition is SCALE_OFFSET_CODEC and data_type_name in _OPEN_FOR_SCALE_OFFSET: + return + codec = _read(definition, data.draw(fields(definition))) + assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) != [] + return + codec = _read(definition, data.draw(fields(definition, configurations))) + assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) == [] + + +Target = tuple[Literal["present", "absent"], str] | tuple[Literal["value"], str, object] +FieldForm = Literal["bare", "object", "configured", "must_understand"] + + +def _configuration_targets( + definition: Definition[Any], +) -> list[Target | tuple[Literal["simplified"]]]: + """What a strategy for `definition` ought to reach, by its configuration's TypedDict.""" + targets: list[Target | tuple[Literal["simplified"]]] = [] + for key, (annotation, required) in typeddict_keys(definition.configuration).members.items(): + if not required: + targets.extend([("present", key), ("absent", key)]) + if get_origin(annotation) is Literal: + targets.extend(("value", key, value) for value in get_args(annotation)) + if definition.canonical is not unchanged: + targets.append(("simplified",)) + return targets + + +def _field_forms(definition: Definition[Any]) -> list[FieldForm]: + """The envelopes the spec allows a field of `definition` to be written in.""" + if definition in _CORE_DATA_TYPES: + return [] # only ever bare, which `test_fields_read` asserts + if definition.requires_configuration: + return ["configured", "must_understand"] + return ["bare", "object", "configured", "must_understand"] + + +def _form_of(field: ZarrV3MetadataFieldJSON) -> set[FieldForm]: + if isinstance(field, str): + return {"bare"} + forms: set[FieldForm] = {"configured" if "configuration" in field else "object"} + if "must_understand" in field: + forms.add("must_understand") + return forms + + +def _hits( + definition: Definition[Any], + target: Target | tuple[Literal["simplified"]], + configuration: Mapping[str, JSONValue], +) -> bool: + match target: + case ("present", key): + return key in configuration + case ("absent", key): + return key not in configuration + case ("value", key, value): + return configuration.get(key) == value + case ("simplified",): + return definition.canonical(configuration) != configuration + + +_REACH = settings(max_examples=2000, database=None) + + +@pytest.mark.parametrize( + ("definition", "target"), + [(d, target) for d in _DEFINITIONS for target in _configuration_targets(d)], + ids=_id, +) +def test_configurations_reach( + definition: Definition[Any], target: Target | tuple[Literal["simplified"]] +) -> None: + configurations = fields(definition).map(lambda field: _read(definition, field).configuration) + find( + configurations, + lambda configuration: _hits(definition, target, configuration), + settings=_REACH, + ) + + +@pytest.mark.parametrize( + ("definition", "form"), + [(d, form) for d in _DEFINITIONS for form in _field_forms(d)], + ids=_id, +) +def test_fields_reach(definition: Definition[Any], form: FieldForm) -> None: + find(fields(definition), lambda field: form in _form_of(field), settings=_REACH) + + +def test_an_added_strategy_is_drawn_from() -> None: + """A strategy joined to the defaults, as an extension author adds one, is the one drawn from.""" + level_three = CodecStrategy(configurations=lambda _chunk: st.just({"level": 3})) + strategies = DEFAULT_STRATEGIES | {GZIP_CODEC: level_three} + field = find(fields(GZIP_CODEC, strategies=strategies), lambda _field: True, settings=_REACH) + assert _read(GZIP_CODEC, field).configuration == {"level": 3} + + +def test_a_definition_without_a_strategy_is_a_lookup_error() -> None: + with pytest.raises(LookupError, match="no strategy for 'sharding_indexed'"): + fields(SHARDING_INDEXED_CODEC) + + +def test_a_strategy_of_another_kind_is_a_type_error() -> None: + strategies: Strategies = {GZIP_CODEC: ChunkKeyEncodingStrategy(configurations=st.just({}))} + with pytest.raises(TypeError, match="draws configurations of a ChunkKeyEncodingDefinition"): + codec_configurations(GZIP_CODEC, strategies=strategies) + + +def test_a_codec_strategy_refusing_an_unknown_chunk_is_a_value_error() -> None: + with pytest.raises(ValueError, match=r"Chunk\(\), a chunk nothing is known of"): + CodecStrategy(configurations=lambda _chunk: None) + + +def test_a_refused_draw_in_the_canonical_spelling_is_a_value_error() -> None: + refused = fields(GZIP_CODEC, st.just({"level": 99}), spellings="canonical") + with pytest.raises(ValueError, match="which is refused"): + find(refused, lambda _field: True, settings=settings(database=None)) + + +def _has_pair(configuration: Mapping[str, Any]) -> bool: + return any( + isinstance(entry, tuple) + for dimension in configuration["chunk_shapes"] + if isinstance(dimension, tuple) + for entry in cast("tuple[object, ...]", dimension) + ) + + +def test_rectilinear_edges_reach_a_run_written_as_a_pair() -> None: + """The `[size, count]` spelling, which no TypedDict target asks for: it is one arm of a union.""" + find(chunk_grid_configurations(RECTILINEAR_CHUNK_GRID), _has_pair, settings=_REACH) + + +FillValueSpelling = Literal[ + "number", "named", "hex lowercase", "hex uppercase", "array", "base64", "NaT" +] + +_FILL_VALUE_SPELLINGS: list[tuple[DataTypeDefinition[Any], FillValueSpelling]] = [ + *( + (definition, spelling) + for definition in ( + FLOAT16_DATA_TYPE, + FLOAT32_DATA_TYPE, + FLOAT64_DATA_TYPE, + COMPLEX64_DATA_TYPE, + COMPLEX128_DATA_TYPE, + ) + for spelling in get_args(FillValueSpelling)[:4] + ), + (BYTES_DATA_TYPE, "array"), + (BYTES_DATA_TYPE, "base64"), + (NUMPY_DATETIME64_DATA_TYPE, "number"), + (NUMPY_DATETIME64_DATA_TYPE, "NaT"), + (NUMPY_TIMEDELTA64_DATA_TYPE, "number"), + (NUMPY_TIMEDELTA64_DATA_TYPE, "NaT"), +] +"""The spellings the specs give a fill value, by the data types whose fill values take them.""" + + +def _spellings_of(value: object) -> set[FillValueSpelling]: + """How a fill value is spelled; a complex one, how either component is.""" + if isinstance(value, bool): + return set() + if isinstance(value, int | float): + return {"number"} + if isinstance(value, tuple): + parts = cast("tuple[object, ...]", value) + if all(isinstance(part, int) for part in parts): + return {"array"} + return {spelling for part in parts for spelling in _spellings_of(part)} + if not isinstance(value, str): + return set() + if value == "NaT": + return {"NaT"} + if value in get_args(FloatSpecialFillValue): + return {"named"} + if value.startswith("0x"): + digits = value[2:] + return { + *(("hex lowercase",) if any(c in "abcdef" for c in digits) else ()), + *(("hex uppercase",) if any(c in "ABCDEF" for c in digits) else ()), + } + return {"base64"} + + +@pytest.mark.parametrize(("definition", "spelling"), _FILL_VALUE_SPELLINGS, ids=_id) +def test_fill_values_reach( + definition: DataTypeDefinition[Any], spelling: FillValueSpelling +) -> None: + values = data_type_configurations(definition).flatmap( + lambda configuration: fill_values(definition, configuration) + ) + find(values, lambda value: spelling in _spellings_of(value), settings=_REACH) diff --git a/packages/zarr-metadata/uv.lock b/packages/zarr-metadata/uv.lock index 07ad9a0ebe..dbcc6a55f3 100644 --- a/packages/zarr-metadata/uv.lock +++ b/packages/zarr-metadata/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" [[package]] @@ -1138,6 +1138,11 @@ dependencies = [ { name = "typing-extensions" }, ] +[package.optional-dependencies] +testing = [ + { name = "hypothesis" }, +] + [package.dev-dependencies] docs = [ { name = "griffe-inherited-docstrings" }, @@ -1155,7 +1160,11 @@ test = [ ] [package.metadata] -requires-dist = [{ name = "typing-extensions", specifier = ">=4.16" }] +requires-dist = [ + { name = "hypothesis", marker = "extra == 'testing'", specifier = ">=6.85" }, + { name = "typing-extensions", specifier = ">=4.16" }, +] +provides-extras = ["testing"] [package.metadata.requires-dev] docs = [ From 3ddacdcc181db779ea0b3285f42170b1497fd21d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 27 Oct 2026 19:25:56 +0100 Subject: [PATCH 2/8] chore(zarr-metadata): number the strategies changelog fragment after #414 Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/changes/{0000.feature.md => 414.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{0000.feature.md => 414.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/0000.feature.md b/packages/zarr-metadata/changes/414.feature.md similarity index 100% rename from packages/zarr-metadata/changes/0000.feature.md rename to packages/zarr-metadata/changes/414.feature.md From 8f726efdc8a781ed6cfa284cb0397a06d65af558 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 31 Oct 2026 20:06:12 +0100 Subject: [PATCH 3/8] chore(zarr-metadata): number the strategies changelog fragment after PR 4575 The fragment was numbered after fork PR 414, so the changelog linked to an unrelated upstream PR. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/changes/{414.feature.md => 4575.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{414.feature.md => 4575.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/414.feature.md b/packages/zarr-metadata/changes/4575.feature.md similarity index 100% rename from packages/zarr-metadata/changes/414.feature.md rename to packages/zarr-metadata/changes/4575.feature.md From 32e19f99b4effe0384b181b1690879ea28fa68f1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 31 Oct 2026 22:57:44 +0100 Subject: [PATCH 4/8] fix(zarr-metadata): draw a fresh empty configuration on every draw `st.just({})` (crc32c) and `st.just(EmptyConfiguration())` (every data type with nothing to configure) handed out one dict for every draw, and `fields` puts the configuration in the field as it is. A test that changed the configuration it drew changed every later draw, which Hypothesis reports as a flaky strategy. Both are now `st.builds(EmptyConfiguration)`. Export `Configurations` and `NOTHING_KNOWN` from `zarr_metadata.testing.strategies`: they appear in the signatures of `DefinitionStrategy.unconstrained` and `codec_configurations`. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/src/zarr_metadata/testing/_codec.py | 4 ++-- .../src/zarr_metadata/testing/_data_type.py | 3 ++- .../src/zarr_metadata/testing/_strategy.py | 1 + .../src/zarr_metadata/testing/strategies.py | 2 ++ packages/zarr-metadata/tests/test_public_api.py | 3 +++ .../zarr-metadata/tests/testing/test_strategies.py | 11 +++++++++++ 6 files changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index df2f0cc820..96262bf06a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -12,7 +12,7 @@ from hypothesis import strategies as st -from zarr_metadata.definition._definition import AcceptedField +from zarr_metadata.definition._definition import AcceptedField, EmptyConfiguration from zarr_metadata.testing._data_type import FLOAT_WIDTHS, INTEGER_RANGES, finite_floats from zarr_metadata.testing._strategy import CodecStrategy from zarr_metadata.v3._definition import Chunk, storage_of @@ -120,7 +120,7 @@ def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> Co CODEC_STRATEGIES: Final[Mapping[CodecDefinition[Any], CodecStrategy]] = { BLOSC_CODEC: _unchunked(_blosc()), BYTES_CODEC: CodecStrategy(configurations=_bytes), - CRC32C_CODEC: _unchunked(st.just({})), + CRC32C_CODEC: _unchunked(st.builds(EmptyConfiguration)), GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py index 64d3b83818..753dd53a0e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -64,7 +64,8 @@ } """The width in bits of each floating-point data type.""" -_EMPTY: Final = st.just(EmptyConfiguration()) +_EMPTY: Final = st.builds(EmptyConfiguration) +"""A fresh empty configuration for each draw, so a test that changes one changes no other.""" _MAX_RAW_BYTES: Final = 32 """The widest raw bits drawn, in bytes: `r256`.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py index 2c1b75852d..23aeb8d6b6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py @@ -122,6 +122,7 @@ def unconstrained(self) -> Configurations: """How to draw each definition's configurations, by the definition.""" __all__ = [ + "NOTHING_KNOWN", "ChunkGridStrategy", "ChunkKeyEncodingStrategy", "CodecStrategy", diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py index d3ab230717..328aebba2f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -258,9 +258,11 @@ def fields( __all__ = [ "DEFAULT_STRATEGIES", + "NOTHING_KNOWN", "ChunkGridStrategy", "ChunkKeyEncodingStrategy", "CodecStrategy", + "Configurations", "DataTypeStrategy", "DefinitionStrategy", "Spellings", diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 5a5de32647..0baad413c2 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -297,6 +297,9 @@ def test_all_is_grouped_and_unique() -> None: # two definitions of one name. "ClaimKey", "Conflict", + # What one definition's configurations are drawn from, in + # `zarr_metadata.testing.strategies`. + "Configurations", "ScopeConflictError", # A reader's problems, as the signatures of every reader name them. "Problems", diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 8747443dba..297980e06b 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -137,6 +137,17 @@ def test_fields_read( assert canonical == field +@pytest.mark.parametrize("definition", _DEFINITIONS, ids=_id) +@given(data=st.data()) +def test_a_changed_draw_changes_no_other(definition: Definition[Any], data: st.DataObject) -> None: + """Each draw is a fresh value: a test that changes the configuration it drew does not change the next draw, which Hypothesis would report as flaky.""" + field = data.draw(fields(definition)) + if isinstance(field, dict) and "configuration" in field: + configuration = cast("dict[str, JSONValue]", field["configuration"]) + assert "changed" not in configuration + configuration["changed"] = True + + @pytest.mark.parametrize("definition", _DATA_TYPES, ids=_id) @given(data=st.data()) def test_fill_values_fit_their_data_type( From da5a0bad139720efe8e154336aec98d8cdd384c8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 31 Oct 2026 23:28:01 +0100 Subject: [PATCH 5/8] fix(zarr-metadata): check every drawn field, and draw hex scale_offset values `fields` read a field only when asked for the canonical spelling, so a caller's configurations the definition refuses were drawn silently in the default spelling. Every field is now read, in a scope built once per call rather than once per draw, and a refused one raises in both spellings. A definition no Zarr v3 scope holds, such as a v2 codec, is a `TypeError` when `fields` is called, not a `StopIteration` on a draw. A float `scale_offset` codec's `offset` and `scale` are fill values of its type, so they are now also drawn as the hex string of their bits. The docstring no longer claims a non-zero scale makes decoding defined: a tiny one still overflows. The codec chunk test now draws chunks of an unknown data type and with unknown lengths, which every codec takes. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../src/zarr_metadata/testing/_codec.py | 21 ++++++-- .../src/zarr_metadata/testing/_data_type.py | 18 ++++++- .../src/zarr_metadata/testing/strategies.py | 46 +++++++++++------ .../tests/testing/test_strategies.py | 49 ++++++++++++++----- 4 files changed, 103 insertions(+), 31 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index 96262bf06a..d4cb2715d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -13,7 +13,12 @@ from hypothesis import strategies as st from zarr_metadata.definition._definition import AcceptedField, EmptyConfiguration -from zarr_metadata.testing._data_type import FLOAT_WIDTHS, INTEGER_RANGES, finite_floats +from zarr_metadata.testing._data_type import ( + FLOAT_WIDTHS, + INTEGER_RANGES, + finite_floats, + float_spellings, +) from zarr_metadata.testing._strategy import CodecStrategy from zarr_metadata.v3._definition import Chunk, storage_of from zarr_metadata.v3.codec.blosc import BLOSC_CNAME, BLOSC_CODEC, BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE @@ -88,8 +93,9 @@ def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | Each is a fill value of the chunk's data type (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), - drawn finite, and the scale non-zero, so that decoding, which divides - by it, is defined. Handed a data type the codec's registry entry does + a float's as a number or the hex string of its bits. Each is drawn + finite, and the scale non-zero, so that decoding never divides by + zero; a scale small enough that decoding overflows is still drawn. Handed a data type the codec's registry entry does not list -- truth values, raw bits, complex numbers, numpy times -- nothing is drawn: for some the spec refuses it, and for the others leaves it open. Handed a chunk whose data type is unknown, the values @@ -105,8 +111,13 @@ def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | offset = st.integers(low, high) scale = st.integers(low, high).filter(lambda value: value != 0) elif definition in FLOAT_WIDTHS: - offset = finite_floats(FLOAT_WIDTHS[definition]) - scale = finite_floats(FLOAT_WIDTHS[definition]).filter(lambda value: value != 0) + width = FLOAT_WIDTHS[definition] + offset = finite_floats(width).flatmap(lambda value: float_spellings(value, width)) + scale = ( + finite_floats(width) + .filter(lambda value: value != 0) + .flatmap(lambda value: float_spellings(value, width)) + ) else: return None return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py index 753dd53a0e..28a0ff687b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -8,6 +8,7 @@ from __future__ import annotations import base64 +import struct from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_args from hypothesis import strategies as st @@ -79,11 +80,20 @@ def finite_floats(width: FloatWidth) -> st.SearchStrategy[float]: return st.floats(width=width, allow_nan=False, allow_infinity=False) +_PACKED: Final[Mapping[FloatWidth, str]] = {16: ">e", 32: ">f", 64: ">d"} +"""How `struct` packs a float of each width, most significant byte first.""" + + def _hex_spellings(digits: str) -> st.SearchStrategy[str]: """`0x` and the digits, the letters in either case.""" return st.sampled_from((f"0x{digits}", f"0x{digits.upper()}")) +def float_spellings(value: float, width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """`value`, a float of `width` bits, as a number or as the hex string of its bits.""" + return st.just(value) | _hex_spellings(struct.pack(_PACKED[width], value).hex()) + + def _float_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: """A number, an integer the type holds exactly, a named non-finite value, or the bits in hex. @@ -173,4 +183,10 @@ def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: } """A strategy for each data type that holds no other field.""" -__all__ = ["DATA_TYPE_STRATEGIES", "FLOAT_WIDTHS", "INTEGER_RANGES", "finite_floats"] +__all__ = [ + "DATA_TYPE_STRATEGIES", + "FLOAT_WIDTHS", + "INTEGER_RANGES", + "finite_floats", + "float_spellings", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py index 328aebba2f..344b67303b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -37,6 +37,7 @@ def test_my_reader_reads_every_blosc_codec(field): from __future__ import annotations +from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast @@ -213,21 +214,30 @@ def _any_spelling( def _scope_of(definition: Definition[Any]) -> ZarrV3Context: - """The scope that claims `definition` and nothing else.""" - slot = next(slot for slot, kind in ZarrV3Context.kinds.items() if isinstance(definition, kind)) - return ZarrV3Context(**cast("ZarrV3ContextSlots", {slot: (definition,)})) + """The scope that claims `definition` and nothing else; a `TypeError` for a definition no slot of a Zarr v3 scope holds.""" + for slot, kind in ZarrV3Context.kinds.items(): + if isinstance(definition, kind): + return ZarrV3Context(**cast("ZarrV3ContextSlots", {slot: (definition,)})) + msg = f"{definition.name!r} is a {type(definition).__name__}, which no slot of a Zarr v3 scope holds" + raise TypeError(msg) -def _canonical( - definition: Definition[Any], field: ZarrV3MetadataFieldJSON +def _checked( + field: ZarrV3MetadataFieldJSON, + *, + definition: Definition[Any], + kind: type[Definition[Any]], + scope: ZarrV3Context, + spellings: Spellings, ) -> ZarrV3MetadataFieldJSON: - """`field` in the spelling `canonicalize` gives it, read in a scope of its own definition.""" - kind = as_kind(kind_of(definition)) - canonical, problems = canonicalize(field, kind, _scope_of(definition)) + """`field`, or its canonical spelling, once it reads without a problem in `scope`; a `ValueError` when it does not.""" + canonical, problems = canonicalize(field, kind, scope) if len(problems) != 0: - msg = f"the strategy for {definition.name!r} drew {field!r}, which is refused: {problems!r}" + msg = f"{field!r}, drawn for {definition.name!r}, which is refused: {problems!r}" raise ValueError(msg) - return cast("ZarrV3MetadataFieldJSON", canonical) + if spellings == "canonical": + return cast("ZarrV3MetadataFieldJSON", canonical) + return field def fields( @@ -244,16 +254,24 @@ def fields( `codec_configurations(definition, chunk)` for a codec handed a chunk, once it is known not to be None. `spellings` says whether every spelling the spec allows is drawn, or only the simplest. + + Each field is read in a scope of its definition alone, and one that + is refused, as one drawn from `configurations` the definition does + not accept is, raises a `ValueError`. """ drawn: Configurations | SearchStrategy[C] = ( _strategy_for(definition, strategies).unconstrained() if configurations is UNSET else configurations ) - spelled = _any_spelling(definition, drawn) - if spellings == "canonical": - return spelled.map(lambda field: _canonical(definition, field)) - return spelled + checked = partial( + _checked, + definition=definition, + kind=as_kind(kind_of(definition)), + scope=_scope_of(definition), + spellings=spellings, + ) + return _any_spelling(definition, drawn).map(checked) __all__ = [ diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 297980e06b..012c584386 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -40,6 +40,7 @@ fields, fill_values, ) +from zarr_metadata.v2.definition import CORE_V2 from zarr_metadata.v3._definition import chunk_grid_lengths from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC @@ -175,11 +176,12 @@ def test_chunk_grids_fit_their_shape( @st.composite def _chunks(draw: st.DrawFn) -> Chunk: - """A chunk of some lengths, of a data type drawn from the strategies.""" - definition = draw(st.sampled_from(_DATA_TYPES)) - data_type = _read(definition, draw(fields(definition))) - lengths = draw(st.lists(st.integers(1, 8), max_size=4)) - return Chunk(tuple(frozenset({length}) for length in lengths), data_type) + """A chunk of a data type drawn from the strategies, or of none known, with lengths known along some axes, all, or none.""" + definition = draw(st.none() | st.sampled_from(_DATA_TYPES)) + data_type = None if definition is None else _read(definition, draw(fields(definition))) + axis = st.none() | st.integers(1, 8).map(lambda length: frozenset({length})) + lengths = draw(st.none() | st.lists(axis, max_size=4).map(tuple)) + return Chunk(lengths, data_type) @pytest.mark.parametrize("definition", _CODECS, ids=_id) @@ -187,12 +189,12 @@ def _chunks(draw: st.DrawFn) -> Chunk: def test_codecs_take_their_chunk( definition: CodecDefinition[Any], data: st.DataObject, chunk: Chunk ) -> None: - """A codec drawn for a chunk takes it; one drawn for none is one the spec does not give it.""" - assert chunk.data_type is not None - assert chunk.data_type.definition is not None - data_type_name = chunk.data_type.definition.name + """A codec drawn for a chunk takes it; one drawn for none is one the spec does not give it, which needs the data type known.""" configurations = codec_configurations(definition, chunk) if configurations is None: + assert chunk.data_type is not None + assert chunk.data_type.definition is not None + data_type_name = chunk.data_type.definition.name event(f"takes no {data_type_name}") if definition is SCALE_OFFSET_CODEC and data_type_name in _OPEN_FOR_SCALE_OFFSET: return @@ -308,12 +310,37 @@ def test_a_codec_strategy_refusing_an_unknown_chunk_is_a_value_error() -> None: CodecStrategy(configurations=lambda _chunk: None) -def test_a_refused_draw_in_the_canonical_spelling_is_a_value_error() -> None: - refused = fields(GZIP_CODEC, st.just({"level": 99}), spellings="canonical") +@pytest.mark.parametrize("spellings", get_args(Spellings)) +def test_a_refused_draw_is_a_value_error(spellings: Spellings) -> None: + refused = fields(GZIP_CODEC, st.just({"level": 99}), spellings=spellings) with pytest.raises(ValueError, match="which is refused"): find(refused, lambda _field: True, settings=settings(database=None)) +def test_a_definition_no_v3_scope_holds_is_a_type_error() -> None: + zlib = CORE_V2.codecs["zlib"] + with pytest.raises(TypeError, match="which no slot of a Zarr v3 scope holds"): + fields(zlib, st.just({})) + + +@pytest.mark.parametrize( + "definition", [FLOAT16_DATA_TYPE, FLOAT32_DATA_TYPE, FLOAT64_DATA_TYPE], ids=_id +) +@pytest.mark.parametrize("member", ["offset", "scale"]) +def test_scale_offset_reaches_a_value_written_in_hex( + definition: DataTypeDefinition[Any], member: str +) -> None: + """A float's `offset` and `scale` are fill values of its type, which the bits in hex spell too.""" + chunk = Chunk(None, _read(definition, definition.name)) + configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) + assert configurations is not None + find( + configurations, + lambda configuration: isinstance(configuration.get(member), str), + settings=_REACH, + ) + + def _has_pair(configuration: Mapping[str, Any]) -> bool: return any( isinstance(entry, tuple) From b042d2657ac264d9e03d94a7996b66bf971a5b1b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 31 Oct 2026 23:37:50 +0100 Subject: [PATCH 6/8] fix(zarr-metadata): draw scale_offset for every number type, and widen Strategies keys `scale_offset` drew nothing for complex numbers, numpy times and data types the package does not define, though its rules refuse only data types whose values are not numbers, and a numpy time "is compatible with any codec that supports arrays of signed 64-bit integers". A numpy time is now drawn as that integer, a complex number as a pair of finite floats, and a type the package does not define takes the identity configuration. None now means only what `CodecStrategy` says it means: no configuration takes the chunk. `Strategies` is keyed by `Any`: a mapping's key type is invariant, so a mapping an extension author builds, keyed by a `CodecDefinition`, was not a `Mapping[Definition[Any], ...]` to pyright. A rectilinear dimension of unknown length may now be drawn with no edges, which its rules accept. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../src/zarr_metadata/testing/_chunk_grid.py | 5 +- .../src/zarr_metadata/testing/_codec.py | 66 ++++++++++++++----- .../src/zarr_metadata/testing/_data_type.py | 14 +++- .../src/zarr_metadata/testing/_strategy.py | 11 +++- .../tests/testing/test_strategies.py | 12 +--- 5 files changed, 75 insertions(+), 33 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py index bbefe58fe3..69dc04fffb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py @@ -65,9 +65,10 @@ def _edges(draw: st.DrawFn, extent: int | None) -> JSONValue: Runs are drawn apart, so two of one size can sit side by side and a pair can count one chunk: spellings the canonical form merges. A - dimension of length zero is covered by no edges at all. + dimension of length zero, or of a length not known, may be covered by + no edges at all. """ - min_runs = 0 if extent == 0 else 1 + min_runs = 0 if extent is None or extent == 0 else 1 runs = draw( st.lists( st.tuples(st.integers(1, _MAX_LENGTH), st.integers(1, _MAX_COUNT)), diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index d4cb2715d4..2d6ec3e6c8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -14,13 +14,16 @@ from zarr_metadata.definition._definition import AcceptedField, EmptyConfiguration from zarr_metadata.testing._data_type import ( + COMPLEX_WIDTHS, FLOAT_WIDTHS, INTEGER_RANGES, + FloatWidth, finite_floats, float_spellings, ) from zarr_metadata.testing._strategy import CodecStrategy from zarr_metadata.v3._definition import Chunk, storage_of +from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS from zarr_metadata.v3.codec.blosc import BLOSC_CNAME, BLOSC_CODEC, BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE from zarr_metadata.v3.codec.bytes import BYTES_CODEC, ENDIANNESS from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC @@ -28,6 +31,9 @@ from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL +from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE if TYPE_CHECKING: from collections.abc import Mapping @@ -46,6 +52,9 @@ _ENDIAN: Final = st.sampled_from(ENDIANNESS) +_NUMPY_TIMES: Final = frozenset({NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE}) +"""The numpy time types, each stored as a signed 64-bit integer.""" + def _bytes(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: """An `endian`, required for a data type whose values take several bytes; None for one whose values vary in size.""" @@ -88,38 +97,61 @@ def _transpose(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]]: return st.integers(0, _MAX_RANK).flatmap(_order) +def _finite_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A finite float of `width` bits, as a number or the hex string of its bits.""" + return finite_floats(width).flatmap(lambda value: float_spellings(value, width)) + + +def _non_zero_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A finite, non-zero float of `width` bits, as a number or the hex string of its bits.""" + return ( + finite_floats(width) + .filter(lambda value: value != 0) + .flatmap(lambda value: float_spellings(value, width)) + ) + + def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: - """An `offset` and a `scale`, each a finite value of the data type the codec is handed, the scale not zero. + """An `offset` and a `scale`, each a finite value of the data type the codec is handed, the scale not zero; None for a data type whose values are not numbers. Each is a fill value of the chunk's data type (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), - a float's as a number or the hex string of its bits. Each is drawn - finite, and the scale non-zero, so that decoding never divides by - zero; a scale small enough that decoding overflows is still drawn. Handed a data type the codec's registry entry does - not list -- truth values, raw bits, complex numbers, numpy times -- - nothing is drawn: for some the spec refuses it, and for the others - leaves it open. Handed a chunk whose data type is unknown, the values - are small non-negative integers, which every listed type holds. + a float's as a number or the hex string of its bits, a complex + number's as a pair of them. Each is drawn finite, and the scale + non-zero, so that decoding never divides by zero; a scale small + enough that decoding overflows is still drawn. A numpy time is drawn + as the signed 64-bit integer it is stored as, since it "is compatible + with any codec that supports arrays of signed 64-bit integers" + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). + A data type this package does not define, whose values it cannot + draw, takes only the configuration with neither, the identity. + Handed a chunk whose data type is unknown, the values are small + non-negative integers, which every integer and float type holds. """ data_type = chunk.data_type definition = data_type.definition if isinstance(data_type, AcceptedField) else None if definition is None: offset: st.SearchStrategy[JSONValue] = st.integers(0, 127) scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) - elif definition in INTEGER_RANGES: - low, high = INTEGER_RANGES[definition] + elif definition.name in NOT_NUMBERS: + return None + elif definition in INTEGER_RANGES or definition in _NUMPY_TIMES: + low, high = INTEGER_RANGES.get(definition, INTEGER_RANGES[INT64_DATA_TYPE]) offset = st.integers(low, high) scale = st.integers(low, high).filter(lambda value: value != 0) elif definition in FLOAT_WIDTHS: - width = FLOAT_WIDTHS[definition] - offset = finite_floats(width).flatmap(lambda value: float_spellings(value, width)) - scale = ( - finite_floats(width) - .filter(lambda value: value != 0) - .flatmap(lambda value: float_spellings(value, width)) + offset = _finite_float_values(FLOAT_WIDTHS[definition]) + scale = _non_zero_float_values(FLOAT_WIDTHS[definition]) + elif definition in COMPLEX_WIDTHS: + width = COMPLEX_WIDTHS[definition] + offset = st.tuples(_finite_float_values(width), _finite_float_values(width)) + # Non-zero in either part is a non-zero complex number. + scale = st.one_of( + st.tuples(_non_zero_float_values(width), _finite_float_values(width)), + st.tuples(_finite_float_values(width), _non_zero_float_values(width)), ) else: - return None + return st.builds(EmptyConfiguration) return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py index 28a0ff687b..a0c043c009 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -65,6 +65,12 @@ } """The width in bits of each floating-point data type.""" +COMPLEX_WIDTHS: Final[Mapping[DataTypeDefinition[Any], FloatWidth]] = { + COMPLEX64_DATA_TYPE: 32, + COMPLEX128_DATA_TYPE: 64, +} +"""The width in bits of each part of each complex data type.""" + _EMPTY: Final = st.builds(EmptyConfiguration) """A fresh empty configuration for each draw, so a test that changes one changes no other.""" @@ -168,8 +174,10 @@ def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: definition: _fill_values_of(_float_fill_values(width)) for definition, width in FLOAT_WIDTHS.items() }, - COMPLEX64_DATA_TYPE: _fill_values_of(_complex_fill_values(32)), - COMPLEX128_DATA_TYPE: _fill_values_of(_complex_fill_values(64)), + **{ + definition: _fill_values_of(_complex_fill_values(width)) + for definition, width in COMPLEX_WIDTHS.items() + }, RAW_BYTES_DATA_TYPE: DataTypeStrategy( configurations=st.integers(1, _MAX_RAW_BYTES).map( lambda size: RawBytesConfiguration(bits=8 * size) @@ -184,9 +192,11 @@ def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: """A strategy for each data type that holds no other field.""" __all__ = [ + "COMPLEX_WIDTHS", "DATA_TYPE_STRATEGIES", "FLOAT_WIDTHS", "INTEGER_RANGES", + "FloatWidth", "finite_floats", "float_spellings", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py index 23aeb8d6b6..9a694832a7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py @@ -13,7 +13,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, cast -from zarr_metadata.definition._definition import Definition from zarr_metadata.v3._definition import ( Chunk, ChunkGridDefinition, @@ -26,6 +25,7 @@ from hypothesis.strategies import SearchStrategy from zarr_metadata._common import JSONValue + from zarr_metadata.definition._definition import Definition Configurations: TypeAlias = "SearchStrategy[Mapping[str, JSONValue]]" """Configurations of one definition, each the mapping every configuration is.""" @@ -118,8 +118,13 @@ def unconstrained(self) -> Configurations: return cast("Configurations", self.configurations(NOTHING_KNOWN)) -Strategies: TypeAlias = Mapping[Definition[Any], DefinitionStrategy] -"""How to draw each definition's configurations, by the definition.""" +Strategies: TypeAlias = Mapping[Any, DefinitionStrategy] +"""How to draw each definition's configurations, by the definition. + +The key is `Any`, not `Definition[Any]`: a mapping's key type is +invariant, so `{MY_CODEC: CodecStrategy(...)}`, a mapping keyed by a +`CodecDefinition`, would not be a `Mapping[Definition[Any], ...]`. +""" __all__ = [ "NOTHING_KNOWN", diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 012c584386..fcd7543967 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -33,7 +33,6 @@ ChunkKeyEncodingStrategy, CodecStrategy, Spellings, - Strategies, chunk_grid_configurations, codec_configurations, data_type_configurations, @@ -92,11 +91,6 @@ _CHUNK_GRIDS = [d for d in _DEFINITIONS if isinstance(d, ChunkGridDefinition)] _CODECS = [d for d in _DEFINITIONS if isinstance(d, CodecDefinition)] -_OPEN_FOR_SCALE_OFFSET = frozenset( - {"complex64", "complex128", "numpy.datetime64", "numpy.timedelta64"} -) -"""Data types `scale_offset` is drawn for none of, though its rules refuse none: its registry entry leaves them open.""" - _SHAPES = st.lists(st.integers(0, 12), max_size=4).map(tuple) @@ -196,8 +190,6 @@ def test_codecs_take_their_chunk( assert chunk.data_type.definition is not None data_type_name = chunk.data_type.definition.name event(f"takes no {data_type_name}") - if definition is SCALE_OFFSET_CODEC and data_type_name in _OPEN_FOR_SCALE_OFFSET: - return codec = _read(definition, data.draw(fields(definition))) assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) != [] return @@ -300,7 +292,9 @@ def test_a_definition_without_a_strategy_is_a_lookup_error() -> None: def test_a_strategy_of_another_kind_is_a_type_error() -> None: - strategies: Strategies = {GZIP_CODEC: ChunkKeyEncodingStrategy(configurations=st.just({}))} + # Unannotated, as an extension author writes it: a mapping keyed by a + # `CodecDefinition` is a `Strategies`. + strategies = {GZIP_CODEC: ChunkKeyEncodingStrategy(configurations=st.just({}))} with pytest.raises(TypeError, match="draws configurations of a ChunkKeyEncodingDefinition"): codec_configurations(GZIP_CODEC, strategies=strategies) From 804d226e02aa01fe1ceb5868e1a8d3d7f87dc2d6 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 31 Oct 2026 23:45:32 +0100 Subject: [PATCH 7/8] refactor(zarr-metadata): share the strategies' empty configuration and rank cap `EMPTY_CONFIGURATIONS` and `MAX_RANK` are defined once, in `_strategy`, rather than once in each module that draws them. A numpy time's `scale_offset` values are now drawn finite, never the integer numpy reads as `NaT`, as the docstring says. The `fields` docstring now says that a codec's chunk rules are not checked, since no chunk is known. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../src/zarr_metadata/testing/_chunk_grid.py | 9 +++---- .../src/zarr_metadata/testing/_codec.py | 24 +++++++++---------- .../src/zarr_metadata/testing/_data_type.py | 9 ++++--- .../src/zarr_metadata/testing/_strategy.py | 9 +++++++ .../src/zarr_metadata/testing/strategies.py | 8 ++++--- 5 files changed, 33 insertions(+), 26 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py index 69dc04fffb..b7528a3907 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py @@ -13,7 +13,7 @@ from hypothesis import strategies as st -from zarr_metadata.testing._strategy import ChunkGridStrategy +from zarr_metadata.testing._strategy import MAX_RANK, ChunkGridStrategy from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID @@ -23,9 +23,6 @@ from zarr_metadata._common import JSONValue from zarr_metadata.v3._definition import ChunkGridDefinition -_MAX_RANK: Final = 5 -"""The most dimensions a grid is drawn with, over no shape.""" - _MAX_LENGTH: Final = 16 """The longest chunk drawn along a dimension whose length is unknown.""" @@ -46,7 +43,7 @@ def _length(extent: int | None) -> st.SearchStrategy[int]: def _regular(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: """A chunk length for each dimension.""" if shape is None: - lengths = st.lists(_length(None), max_size=_MAX_RANK).map(tuple) + lengths = st.lists(_length(None), max_size=MAX_RANK).map(tuple) else: lengths = st.tuples(*(_length(extent) for extent in shape)) return lengths.map(lambda chunk_shape: {"chunk_shape": chunk_shape}) @@ -93,7 +90,7 @@ def _rectilinear_dimension(extent: int | None) -> st.SearchStrategy[JSONValue]: def _rectilinear(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: """Inline chunk shapes, one entry for each dimension.""" if shape is None: - dimensions = st.lists(_rectilinear_dimension(None), max_size=_MAX_RANK).map(tuple) + dimensions = st.lists(_rectilinear_dimension(None), max_size=MAX_RANK).map(tuple) else: dimensions = st.tuples(*(_rectilinear_dimension(extent) for extent in shape)) return dimensions.map(lambda chunk_shapes: {"kind": "inline", "chunk_shapes": chunk_shapes}) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index 2d6ec3e6c8..caf7b71a05 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -12,7 +12,7 @@ from hypothesis import strategies as st -from zarr_metadata.definition._definition import AcceptedField, EmptyConfiguration +from zarr_metadata.definition._definition import AcceptedField from zarr_metadata.testing._data_type import ( COMPLEX_WIDTHS, FLOAT_WIDTHS, @@ -21,7 +21,7 @@ finite_floats, float_spellings, ) -from zarr_metadata.testing._strategy import CodecStrategy +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, MAX_RANK, CodecStrategy from zarr_metadata.v3._definition import Chunk, storage_of from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS from zarr_metadata.v3.codec.blosc import BLOSC_CNAME, BLOSC_CODEC, BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE @@ -31,7 +31,6 @@ from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL -from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE @@ -41,9 +40,6 @@ from zarr_metadata._common import JSONValue from zarr_metadata.v3._definition import CodecDefinition -_MAX_RANK: Final = 5 -"""The most axes an `order` is drawn with, when the chunk does not say.""" - _BLOSC_MAX_TYPESIZE: Final = 255 """The widest element c-blosc takes: `BLOSC_MAX_TYPESIZE`.""" @@ -55,6 +51,9 @@ _NUMPY_TIMES: Final = frozenset({NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE}) """The numpy time types, each stored as a signed 64-bit integer.""" +_NUMPY_TIME_RANGE: Final = (-(2**63) + 1, 2**63 - 1) +"""The signed 64-bit integers but the least, which numpy reads as `NaT`: the finite numpy times.""" + def _bytes(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: """An `endian`, required for a data type whose values take several bytes; None for one whose values vary in size.""" @@ -94,7 +93,7 @@ def _transpose(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]]: """A permutation of the chunk's axes; of some number of axes, when the chunk does not say.""" if chunk.rank is not None: return _order(chunk.rank) - return st.integers(0, _MAX_RANK).flatmap(_order) + return st.integers(0, MAX_RANK).flatmap(_order) def _finite_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: @@ -120,8 +119,9 @@ def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | number's as a pair of them. Each is drawn finite, and the scale non-zero, so that decoding never divides by zero; a scale small enough that decoding overflows is still drawn. A numpy time is drawn - as the signed 64-bit integer it is stored as, since it "is compatible - with any codec that supports arrays of signed 64-bit integers" + as the signed 64-bit integer it is stored as, never `NaT`, since it + "is compatible with any codec that supports arrays of signed 64-bit + integers" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). A data type this package does not define, whose values it cannot draw, takes only the configuration with neither, the identity. @@ -136,7 +136,7 @@ def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | elif definition.name in NOT_NUMBERS: return None elif definition in INTEGER_RANGES or definition in _NUMPY_TIMES: - low, high = INTEGER_RANGES.get(definition, INTEGER_RANGES[INT64_DATA_TYPE]) + low, high = INTEGER_RANGES.get(definition, _NUMPY_TIME_RANGE) offset = st.integers(low, high) scale = st.integers(low, high).filter(lambda value: value != 0) elif definition in FLOAT_WIDTHS: @@ -151,7 +151,7 @@ def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | st.tuples(_finite_float_values(width), _non_zero_float_values(width)), ) else: - return st.builds(EmptyConfiguration) + return EMPTY_CONFIGURATIONS return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) @@ -163,7 +163,7 @@ def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> Co CODEC_STRATEGIES: Final[Mapping[CodecDefinition[Any], CodecStrategy]] = { BLOSC_CODEC: _unchunked(_blosc()), BYTES_CODEC: CodecStrategy(configurations=_bytes), - CRC32C_CODEC: _unchunked(st.builds(EmptyConfiguration)), + CRC32C_CODEC: _unchunked(EMPTY_CONFIGURATIONS), GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py index a0c043c009..b23a79b605 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -14,8 +14,7 @@ from hypothesis import strategies as st from zarr_metadata._numpy import NUMPY_TIME_MAX_SCALE_FACTOR -from zarr_metadata.definition._definition import EmptyConfiguration -from zarr_metadata.testing._strategy import DataTypeStrategy +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, DataTypeStrategy from zarr_metadata.v3.data_type._float import FloatSpecialFillValue from zarr_metadata.v3.data_type._numpy_time import NUMPY_TIME_UNIT from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE @@ -71,8 +70,6 @@ } """The width in bits of each part of each complex data type.""" -_EMPTY: Final = st.builds(EmptyConfiguration) -"""A fresh empty configuration for each draw, so a test that changes one changes no other.""" _MAX_RAW_BYTES: Final = 32 """The widest raw bits drawn, in bytes: `r256`.""" @@ -120,7 +117,9 @@ def _float_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: def _fill_values_of(values: st.SearchStrategy[JSONValue]) -> DataTypeStrategy: """A data type with nothing to configure, whose fill values are `values`.""" - return DataTypeStrategy(configurations=_EMPTY, fill_values=lambda _configuration: values) + return DataTypeStrategy( + configurations=EMPTY_CONFIGURATIONS, fill_values=lambda _configuration: values + ) def _byte_values(size: int) -> st.SearchStrategy[JSONValue]: diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py index 9a694832a7..73de0f8f5f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py @@ -13,6 +13,9 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, cast +from hypothesis import strategies as st + +from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( Chunk, ChunkGridDefinition, @@ -33,6 +36,12 @@ NOTHING_KNOWN: Final = Chunk() """A chunk nothing is known of, which every codec takes.""" +MAX_RANK: Final = 5 +"""The most axes drawn where the number of axes is not known.""" + +EMPTY_CONFIGURATIONS: Final = st.builds(EmptyConfiguration) +"""A fresh empty configuration for each draw, so a test that changes one changes no other.""" + class DefinitionStrategy(ABC): """How to draw the configurations of one kind of definition.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py index 344b67303b..8a37ebf0b4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -255,9 +255,11 @@ def fields( once it is known not to be None. `spellings` says whether every spelling the spec allows is drawn, or only the simplest. - Each field is read in a scope of its definition alone, and one that - is refused, as one drawn from `configurations` the definition does - not accept is, raises a `ValueError`. + Each field is read in a scope of its definition alone, with no chunk + known, and one its definition's rules refuse, as they refuse one + drawn from `configurations` they do not accept, raises a + `ValueError`. A codec's chunk rules are not checked: what fits the + chunk is what `codec_configurations` draws. """ drawn: Configurations | SearchStrategy[C] = ( _strategy_for(definition, strategies).unconstrained() From 053731e07351810f07f8ea66d609f4d00332e240 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 31 Oct 2026 23:50:33 +0100 Subject: [PATCH 8/8] test(zarr-metadata): cover scale_offset over a data type the package does not define The identity configuration `scale_offset` draws for a data type the package does not define had no test, since the chunk test draws only the package's own data types. A test now reads an extension `bfloat16` in a scope of its own and checks the codec takes every draw. Also splits the `_scope_of` error message across two literals and drops a doubled blank line in `_data_type`. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../src/zarr_metadata/testing/_data_type.py | 1 - .../src/zarr_metadata/testing/strategies.py | 5 ++++- .../tests/testing/test_strategies.py | 21 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py index b23a79b605..dd06df28a0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -70,7 +70,6 @@ } """The width in bits of each part of each complex data type.""" - _MAX_RAW_BYTES: Final = 32 """The widest raw bits drawn, in bytes: `r256`.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py index 8a37ebf0b4..fd13c43839 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -218,7 +218,10 @@ def _scope_of(definition: Definition[Any]) -> ZarrV3Context: for slot, kind in ZarrV3Context.kinds.items(): if isinstance(definition, kind): return ZarrV3Context(**cast("ZarrV3ContextSlots", {slot: (definition,)})) - msg = f"{definition.name!r} is a {type(definition).__name__}, which no slot of a Zarr v3 scope holds" + msg = ( + f"{definition.name!r} is a {type(definition).__name__}, " + "which no slot of a Zarr v3 scope holds" + ) raise TypeError(msg) diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index fcd7543967..c0d8aa0e3d 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -24,6 +24,7 @@ from zarr_metadata.definition import ( AcceptedField, Definition, + EmptyConfiguration, fill_value_problems, resolve, ) @@ -63,6 +64,7 @@ ChunkGridDefinition, CodecDefinition, DataTypeDefinition, + ZarrV3Context, ) if TYPE_CHECKING: @@ -311,6 +313,25 @@ def test_a_refused_draw_is_a_value_error(spellings: Spellings) -> None: find(refused, lambda _field: True, settings=settings(database=None)) +_BFLOAT16 = DataTypeDefinition(name="bfloat16", configuration=EmptyConfiguration) +"""A number type this package does not define, as an extension author would.""" + + +@given(data=st.data()) +def test_scale_offset_takes_a_data_type_the_package_does_not_define(data: st.DataObject) -> None: + """Values of a type the package does not define cannot be drawn, so the configuration is the identity, which its rules take.""" + data_type, problems = resolve( + "bfloat16", DataTypeDefinition, ZarrV3Context(data_types=[_BFLOAT16]) + ) + assert problems == () + chunk = Chunk(None, data_type) + configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) + assert configurations is not None + configuration = data.draw(configurations) + assert configuration == {} + assert list(SCALE_OFFSET_CODEC.chunk_rules(configuration, {}, chunk)) == [] + + def test_a_definition_no_v3_scope_holds_is_a_type_error() -> None: zlib = CORE_V2.codecs["zlib"] with pytest.raises(TypeError, match="which no slot of a Zarr v3 scope holds"):