diff --git a/changes/4616.bugfix.md b/changes/4616.bugfix.md new file mode 100644 index 0000000000..5745d77c0f --- /dev/null +++ b/changes/4616.bugfix.md @@ -0,0 +1,5 @@ +`zarr migrate v3` now migrates Zarr format 2 arrays whose `fill_value` is +`null`, such as the integer variables and coordinates xarray writes. The v3 +metadata uses the data type's default fill value, which is what reading an +unwritten chunk of the v2 array already returned. Before, the migration failed +with `TypeError: Cannot convert object None ... to a scalar`. diff --git a/changes/4626.removal.md b/changes/4626.removal.md new file mode 100644 index 0000000000..34c208c0ba --- /dev/null +++ b/changes/4626.removal.md @@ -0,0 +1 @@ +The minimum supported packaging version is now 24.0. diff --git a/ci/big-endian/Dockerfile b/ci/big-endian/Dockerfile index b473c47e43..a2700103ba 100644 --- a/ci/big-endian/Dockerfile +++ b/ci/big-endian/Dockerfile @@ -32,7 +32,7 @@ RUN git clone -q --depth 1 --branch 1.1.2 https://github.com/google/crc32c /tmp/ RUN python3 -m venv --system-site-packages /venv ENV PATH=/venv/bin:$PATH -RUN pip install -q --no-binary google-crc32c google-crc32c msgspec pytest-accept +RUN pip install -q --no-binary google-crc32c google-crc32c msgspec # Debian sid's python3-dateutil 3.0.0 declares a dependency on the `tzdata` distribution, # which no Debian package installs, so `pip check` fails on the image as built. tzdata is # pure Python; install it from PyPI. diff --git a/ci/min_deps-overrides.txt b/ci/min_deps-overrides.txt deleted file mode 100644 index 2eb5ec49de..0000000000 --- a/ci/min_deps-overrides.txt +++ /dev/null @@ -1,3 +0,0 @@ -# pytest>=9.2 requires packaging>=24, but min_deps must keep testing the -# declared runtime floor packaging>=22.0, so this env stays on pytest<9.2. -pytest<9.2 diff --git a/docs/user-guide/installation.md b/docs/user-guide/installation.md index a7487e83c8..4ae4baa76a 100644 --- a/docs/user-guide/installation.md +++ b/docs/user-guide/installation.md @@ -5,7 +5,7 @@ Required dependencies include: - [Python](https://docs.python.org/3/) (3.12 or later) -- [packaging](https://packaging.pypa.io/en/stable/) (22.0 or later) +- [packaging](https://packaging.pypa.io/en/stable/) (24.0 or later) - [numpy](https://numpy.org) (2.0 or later) - [numcodecs](https://numcodecs.readthedocs.io/en/stable/) (0.14 or later) - [google-crc32c](https://github.com/googleapis/python-crc32c) (1.5 or later) diff --git a/packages/zarr-metadata/changes/4628.bugfix.md b/packages/zarr-metadata/changes/4628.bugfix.md new file mode 100644 index 0000000000..07af0924ff --- /dev/null +++ b/packages/zarr-metadata/changes/4628.bugfix.md @@ -0,0 +1,10 @@ +**Breaking:** `cast_value` and `scale_offset` refuse the data types their +specs do not list. Each spec names the integer and floating-point types as +"the following data types defined in this repository" that it supports, and +that repository also defines the numpy time types and the complex types. So +both codecs now refuse `numpy.datetime64` and `numpy.timedelta64`, and +`cast_value` refuses them as the data type it casts to, and `scale_offset` +now refuses `complex64` and `complex128`. All of these were accepted. The time +types' own spec says they are "compatible with any codec that supports arrays +of signed 64-bit integers", which the two codecs' specs do not bear out. +`zarr_metadata.testing.strategies` no longer draws these combinations. diff --git a/packages/zarr-metadata/changes/4628.feature.md b/packages/zarr-metadata/changes/4628.feature.md new file mode 100644 index 0000000000..0765e5b880 --- /dev/null +++ b/packages/zarr-metadata/changes/4628.feature.md @@ -0,0 +1,21 @@ +**Breaking:** a v3 codec is defined with the class of its kind, +`ArrayArrayCodecDefinition`, `ArrayBytesCodecDefinition` or +`BytesBytesCodecDefinition`, instead of `CodecDefinition(kind=...)`. The kind +is now the class's `kind` property, and `CodecDefinition` is their abstract +base, which a scope and `resolve` still use to name the kind. A codec that +receives an array, an `ArrayCodecDefinition`, has `chunk_rules` and +`pipelines`, an array -> array codec also has `transition`, and a bytes -> +bytes codec, which receives bytes, has none of them, so building one with any +of them is a `TypeError` from the class itself. + +A v3 data type definition declares what kind of numbers its values are with +`number_class`, a `NumberClass`: `"integral"`, `"floating_point"`, +`"complex"` or `"non_numeric"`. `cast_value` and `scale_offset` read it to +decide which data types they accept, where each kept a list of data type +names, so a data type defined elsewhere is judged by what it declares: both +codecs refuse one that declares complex or non-numeric values, and +`cast_value` refuses to `wrap` to one that declares floating-point values. A +data type that declares no number class is accepted by both, as before. A data +type defined elsewhere under one of the float names the `cast_value` spec +lists, such as `bfloat16`, was refused as a `wrap` target by its name, and is +now refused once it declares `"floating_point"`. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index 5604ce5471..57acdeffbd 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -33,7 +33,7 @@ from typing_extensions import TypedDict from zarr_metadata import ValidationProblem from zarr_metadata.definition import Nested -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, BytesBytesCodecDefinition, ZarrV3Context class AcmeLz4Configuration(TypedDict, closed=True): @@ -51,10 +51,9 @@ def acme_lz4_rules( ) -ACME_LZ4 = CodecDefinition( +ACME_LZ4 = BytesBytesCodecDefinition( name="acme.lz4", configuration=AcmeLz4Configuration, - kind="bytes_bytes", size="dynamic", rules=acme_lz4_rules, ) @@ -110,13 +109,20 @@ Other kinds declare more: `create_default` is given no codecs (`array_bytes_codec_default`, such as `vlen-utf8` for `string`). A data type whose values vary in size declares its own, since the default, `bytes`, encodes values of a fixed - size. + size. It declares what kind of numbers its values are (`number_class`: + `"integral"`, `"floating_point"`, `"complex"` or `"non_numeric"`), which + the codecs that are defined only for some kinds of numbers read: + `cast_value` and `scale_offset` refuse a data type that declares + complex or non-numeric values. A data type that does not declare it is + accepted by both. - A chunk grid declares which shapes it fits (`shape_rules`) and its chunk lengths (`chunk_lengths`). -- A codec declares: - - what it checks about the chunk it receives (`chunk_rules`), - - what it hands on (`transition`), - - any pipelines it contains (`pipelines`). +- A codec is defined by the class of its kind, `ArrayArrayCodecDefinition`, + `ArrayBytesCodecDefinition` or `BytesBytesCodecDefinition`. A codec that + receives an array declares what it checks about the chunk it receives + (`chunk_rules`) and any pipelines it contains (`pipelines`), and an + array -> array codec also declares what it hands on (`transition`). A + bytes -> bytes codec receives bytes, so it declares none of these. The [`zarr_metadata.v3.definition`](api/v3/definition.md) docstring describes each of these. diff --git a/packages/zarr-metadata/docs/parsing-validation.md b/packages/zarr-metadata/docs/parsing-validation.md index abdea3c790..1cc0ff2a2f 100644 --- a/packages/zarr-metadata/docs/parsing-validation.md +++ b/packages/zarr-metadata/docs/parsing-validation.md @@ -258,12 +258,12 @@ To make an unknown data type or codec known, add its definition to the context's ```python test="true" session="parsing-validation" from zarr_metadata.definition import EmptyConfiguration -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, BytesBytesCodecDefinition, ZarrV3Context # The definition of the codec that was unknown: here, a codec named # `acme.codec` that takes no configuration. -MY_CODEC = CodecDefinition( - name="acme.codec", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" +MY_CODEC = BytesBytesCodecDefinition( + name="acme.codec", configuration=EmptyConfiguration, size="dynamic" ) context = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[MY_CODEC])) reading = read_array_metadata_v3(raw, context=context) diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 849023c493..4dc31e9162 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -74,7 +74,7 @@ resolved, problems = resolve({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) - resolved # RefusedDeclaration(..., definition=CodecDefinition(name='gzip'), ...) + resolved # RefusedDeclaration(..., definition=BytesBytesCodecDefinition(name='gzip'), ...) problems[0].loc # ('configuration', 'level') problems[0].input # 12 dict(problems[0].ctx) # {'ge': 0, 'le': 9} @@ -125,7 +125,7 @@ from zarr_metadata import ValidationProblem from zarr_metadata.definition import Nested -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, BytesBytesCodecDefinition, ZarrV3Context class AcmeLz4Configuration(TypedDict, closed=True): @@ -143,10 +143,9 @@ def acme_lz4_rules( ) -ACME_LZ4 = CodecDefinition( +ACME_LZ4 = BytesBytesCodecDefinition( name="acme.lz4", configuration=AcmeLz4Configuration, - kind="bytes_bytes", size="dynamic", rules=acme_lz4_rules, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index 2b26c320dd..5ec45096bd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -22,7 +22,6 @@ from zarr_metadata._common import EmptyConfiguration from zarr_metadata.definition._definition import AcceptedDeclaration from zarr_metadata.testing._data_type import ( - COMPLEX_WIDTHS, FLOAT_WIDTHS, INTEGER_RANGES, float_fill_values, @@ -35,10 +34,11 @@ fill_values_of, pipelines_for, read, + scope_of, ) from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, MAX_RANK, CodecStrategy from zarr_metadata.v3._definition import Chunk, DataTypeDefinition, storage_of -from zarr_metadata.v3.codec._arithmetic import FLOATING_POINT, NO_REAL_NUMBERS, NOT_NUMBERS +from zarr_metadata.v3.codec._arithmetic import NO_REAL_NUMBERS from zarr_metadata.v3.codec.blosc import ( BLOSC_CNAME, BLOSC_CODEC, @@ -72,8 +72,6 @@ ZSTD_MIN_LEVEL, ZstdCodecConfiguration, ) -from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE -from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE if TYPE_CHECKING: from collections.abc import Mapping @@ -81,7 +79,13 @@ from zarr_metadata._common import JSONValue from zarr_metadata.definition._definition import ResolvedDeclaration from zarr_metadata.testing._strategy import Configurations, Strategies - from zarr_metadata.v3._definition import CodecDefinition, CodecSize, Lengths + from zarr_metadata.v3._definition import ( + ArrayBytesCodecDefinition, + CodecDefinition, + CodecSize, + Lengths, + NumberClass, + ) _BLOSC_MAX_TYPESIZE: Final = 255 """The widest element c-blosc takes: `BLOSC_MAX_TYPESIZE`.""" @@ -109,12 +113,6 @@ _SMALL_NUMBERS: Final = st.integers(0, 127) """Values every integral and floating-point data type holds: for a chunk whose data type is not known.""" -_NUMPY_TIMES: Final = frozenset({NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE}) -"""The numpy time types, each stored as a signed 64-bit integer.""" - -_NUMPY_TIME_RANGE: Final = (-(2**63) + 1, 2**63 - 1) -"""The signed 64-bit integers but the least, which numpy reads as `NaT`: the finite numpy times.""" - @st.composite def _endians(draw: st.DrawFn, *, required: bool) -> BytesCodecConfiguration: @@ -183,18 +181,14 @@ def _offsets_and_scales( def _scale_offset( chunk: Chunk, _strategies: Strategies ) -> st.SearchStrategy[ScaleOffsetCodecConfiguration] | None: - """An `offset` and a `scale`, each a value of the data type the codec is handed, the scale not zero; None for a data type whose values are not numbers. + """An `offset` and a `scale`, each a value of the data type the codec is handed, the scale not zero; None for a data type whose values are not integers or floating-point numbers. Each is a fill value of the chunk's data type (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), so a float's is in every spelling a fill value takes, a `NaN` and an - infinity among them, and a complex number's is a pair of them. The - scale is drawn non-zero in any spelling, so that decoding never divides - by zero; a scale small enough that decoding overflows is still drawn. A - numpy time is drawn as the signed 64-bit integer it is stored as, never - `NaT`, since it "is compatible with any codec that supports arrays of - signed 64-bit integers" - (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). + infinity among them. The scale is drawn non-zero in any spelling, so + that decoding never divides by zero; a scale small enough that decoding + overflows is still drawn. A data type this package does not define, whose values it cannot draw, takes only the configuration with neither, the identity. Handed a chunk whose data type is unknown, the values are small @@ -205,22 +199,15 @@ def _scale_offset( if definition is None: offset: st.SearchStrategy[JSONValue] = _SMALL_NUMBERS scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) - elif definition.name in NOT_NUMBERS: + elif definition.number_class in NO_REAL_NUMBERS: return None - elif definition in INTEGER_RANGES or definition in _NUMPY_TIMES: - low, high = INTEGER_RANGES.get(definition, _NUMPY_TIME_RANGE) + elif definition in INTEGER_RANGES: + low, high = INTEGER_RANGES[definition] offset = st.integers(low, high) scale = st.integers(low, high).filter(lambda value: value != 0) elif definition in FLOAT_WIDTHS: offset = float_fill_values(FLOAT_WIDTHS[definition]) scale = offset.filter(lambda value: not is_zero_float(value)) - elif definition in COMPLEX_WIDTHS: - part = float_fill_values(COMPLEX_WIDTHS[definition]) - offset = st.tuples(part, part) - # Non-zero in either part is a non-zero complex number. - scale = st.tuples(part, part).filter( - lambda pair: not (is_zero_float(pair[0]) and is_zero_float(pair[1])) - ) else: return st.builds(ScaleOffsetCodecConfiguration) return _offsets_and_scales(offset, scale) @@ -235,7 +222,7 @@ def _zstd(draw: st.DrawFn) -> ZstdCodecConfiguration: return configuration -def _vlen(definition: CodecDefinition[Any]) -> CodecStrategy: +def _vlen(definition: ArrayBytesCodecDefinition[Any]) -> CodecStrategy: """A codec with nothing to configure that takes only some data types: none for a chunk of another, which its own chunk rules say.""" return CodecStrategy( configurations=lambda chunk, _strategies: ( @@ -251,9 +238,11 @@ def _unchunked(configurations: Configurations) -> CodecStrategy: return CodecStrategy(configurations=lambda _chunk, _strategies: configurations) -def _name_read(data_type: ResolvedDeclaration[DataTypeDefinition[Any]] | None) -> str | None: - """The name of the definition that accepted `data_type`; None when none did.""" - return data_type.definition.name if isinstance(data_type, AcceptedDeclaration) else None +def _number_class_read( + data_type: ResolvedDeclaration[DataTypeDefinition[Any]] | None, +) -> NumberClass | None: + """The number class the definition that accepted `data_type` declares; None when none accepted it, or it declares none.""" + return data_type.definition.number_class if isinstance(data_type, AcceptedDeclaration) else None def _source_scalars(chunk: Chunk, strategies: Strategies) -> st.SearchStrategy[JSONValue]: @@ -294,7 +283,7 @@ def _cast_value_configuration( declaration, DataTypeDefinition, strategies ) modes: tuple[CastOutOfRangeMode, ...] = CAST_OUT_OF_RANGE_MODE - if _name_read(target) in FLOATING_POINT: + if _number_class_read(target) == "floating_point": modes = tuple(mode for mode in modes if mode != "wrap") configuration = CastValueCodecConfiguration(data_type=declaration) if draw(st.booleans()): @@ -320,9 +309,14 @@ def _cast_value( of its direction; `wrap` is drawn only for a type that is not floating-point. """ - if _name_read(chunk.data_type) in NO_REAL_NUMBERS: + if _number_class_read(chunk.data_type) in NO_REAL_NUMBERS: return None - targets = [name for name in data_type_names(strategies) if name not in NO_REAL_NUMBERS] + data_types = scope_of(strategies).data_types + targets = [ + name + for name in data_type_names(strategies) + if data_types[name].number_class not in NO_REAL_NUMBERS + ] if len(targets) == 0: return None return _cast_value_configuration(targets, _source_scalars(chunk, strategies), strategies) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py index 6c8c2094d2..f2c0ec3c0b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py @@ -12,7 +12,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, TypeVar, cast from hypothesis import reject from hypothesis import strategies as st @@ -21,7 +21,15 @@ from zarr_metadata.testing._strategy import NOTHING_KNOWN from zarr_metadata.v3._context import ZarrV3Context from zarr_metadata.v3._core import CORE_CONTEXT -from zarr_metadata.v3._definition import CodecDefinition, DataTypeDefinition, no_pipelines +from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, + ArrayBytesCodecDefinition, + ArrayCodecDefinition, + BytesBytesCodecDefinition, + CodecDefinition, + DataTypeDefinition, + no_pipelines, +) if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Sequence @@ -39,7 +47,10 @@ ) from zarr_metadata.v3 import ZarrV3DeclarationJSON from zarr_metadata.v3._context import ZarrV3ContextSlots - from zarr_metadata.v3._definition import Chunk, CodecKind, CodecSize + from zarr_metadata.v3._definition import Chunk, CodecSize + +K = TypeVar("K", bound="CodecDefinition[Any]") +"""A kind of codec: the class of its definitions.""" _MAX_ARRAY_ARRAY: Final = 2 """The most array -> array codecs a pipeline is drawn with.""" @@ -221,20 +232,20 @@ def fill_values_of( def _taking( chunk: Chunk, - kind: CodecKind, + kind: type[K], sizes: frozenset[CodecSize], strategies: Strategies, *, holding: bool = True, -) -> list[tuple[CodecDefinition[Any], Configurations]]: +) -> list[tuple[K, Configurations]]: """Each codec of `kind` and one of `sizes` that `strategies` draw for, that takes `chunk`, with its configurations. Without `holding`, a codec that holds pipelines of its own is left out: what one of those takes depends on what its pipelines do. """ - taking: list[tuple[CodecDefinition[Any], Configurations]] = [] + taking: list[tuple[K, Configurations]] = [] for definition in sorted(scope_of(strategies).codecs.values(), key=lambda d: d.name): - if definition.kind != kind or definition.size not in sizes: + if not isinstance(definition, kind) or definition.size not in sizes: continue if not holding and _holds_pipelines(definition): continue @@ -246,7 +257,7 @@ def _taking( def _holds_pipelines(definition: CodecDefinition[Any]) -> bool: - return definition.pipelines is not no_pipelines + return isinstance(definition, ArrayCodecDefinition) and definition.pipelines is not no_pipelines def pipelines_for( @@ -258,15 +269,15 @@ def pipelines_for( that holds pipelines of its own takes a chunk only if what it holds does, so it settles nothing. """ - if len(_taking(chunk, "array_bytes", sizes, strategies, holding=False)) == 0: + if len(_taking(chunk, ArrayBytesCodecDefinition, sizes, strategies, holding=False)) == 0: return None return _walk(chunk, strategies, sizes) @st.composite def _codec( - draw: st.DrawFn, options: Sequence[tuple[CodecDefinition[Any], Configurations]] -) -> tuple[CodecDefinition[Any], ZarrV3DeclarationJSON]: + draw: st.DrawFn, options: Sequence[tuple[K, Configurations]] +) -> tuple[K, ZarrV3DeclarationJSON]: definition, configurations = draw(st.sampled_from(options)) return definition, draw(any_spelling(definition, configurations)) @@ -285,7 +296,7 @@ def _walk( codecs: list[ZarrV3DeclarationJSON] = [] handed = chunk for _ in range(draw(st.integers(0, _MAX_ARRAY_ARRAY))): - options = _taking(handed, "array_array", sizes, strategies) + options = _taking(handed, ArrayArrayCodecDefinition, sizes, strategies) if len(options) == 0: break definition, declaration = draw(_codec(options)) @@ -293,11 +304,11 @@ def _walk( assert isinstance(codec, AcceptedDeclaration) handed = definition.transition(codec.configuration, codec.nested, handed) codecs.append(declaration) - options = _taking(handed, "array_bytes", sizes, strategies) + options = _taking(handed, ArrayBytesCodecDefinition, sizes, strategies) if len(options) == 0: reject() # what the array -> array codecs handed on, no array -> bytes codec takes codecs.append(draw(_codec(options))[1]) - options = _taking(NOTHING_KNOWN, "bytes_bytes", sizes, strategies) + options = _taking(NOTHING_KNOWN, BytesBytesCodecDefinition, sizes, strategies) if len(options) != 0: count = draw(st.integers(0, _MAX_BYTES_BYTES)) codecs.extend(draw(_codec(options))[1] for _ in range(count)) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index a8171b7e61..0d734d81e1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -11,8 +11,8 @@ from __future__ import annotations -import dataclasses import re +from abc import ABC, abstractmethod from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass from typing import ( @@ -127,6 +127,24 @@ def unknown_storage(*_: object) -> None: """The storage of a data type that does not declare it: unknown.""" +NumberClass = Literal["integral", "floating_point", "complex", "non_numeric"] +"""What kind of numbers a data type's values are: integers, floating-point numbers, complex numbers, or values that are not plain numbers. + +A codec that is defined only for some kinds of numbers reads this. For +example, `cast_value` "is only defined for data types that model real +numbers: floating-point and integral data types" +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L5-L9), +and its `wrap` mode is "Only permitted when data_type is an integral +type" +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L84). +`non_numeric` is every other kind of value: truth values, times, text, +byte strings, records and lists, and a value that may be missing, such +as an optional integer, whose values are an integer or nothing. A record +or a list of numbers, or an optional number, holds numbers but is not one, +so the codecs that read the number class treat it as they treat text. +""" + + def little_endian_bytes_codec(*_: object) -> BytesCodecObject: """The array -> bytes codec of a data type that does not declare one: `bytes` with little `endian`, which encodes values of any fixed size.""" return {"name": BYTES_CODEC_NAME, "configuration": {"endian": "little"}} @@ -197,6 +215,13 @@ class DataTypeDefinition(WithFillValue[C], kind=True, format=3): any fixed size, so a data type whose values vary in size declares its own. + `number_class` declares what kind of numbers the values are: integral, + floating-point, complex, or not plain numbers. A codec that is defined only + for some kinds of numbers reads it, as `cast_value` reads it to accept + only data types that model real numbers. A data type that does not + declare it leaves its number class unknown, and such a codec accepts + it, since nothing is known of its values. + A data type named the way a document writes raw bits of one size (such as `r16`) is rejected. That name is read as `r*`, so this definition would never be used. @@ -235,8 +260,16 @@ def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JS little_endian_bytes_codec ) """The array -> bytes codec that encodes the values of an array built without codecs, such as `vlen-utf8` for `string`.""" + number_class: NumberClass | None = None + """What kind of numbers the values are, such as `"integral"` for `int8`; None when unknown.""" def _refusal(self) -> str | None: + number_class = cast("object", self.number_class) + if number_class is not None and number_class not in get_args(NumberClass): + return ( + f"{self.name!r}: number_class is one of {get_args(NumberClass)!r} or None, " + f"got {number_class!r}" + ) if RAW_BYTES_NAME_PATTERN.fullmatch(self.name) is not None: return ( f"{self.name!r} is how a document writes raw bits of one size, which read as " @@ -300,48 +333,18 @@ class ChunkKeyEncodingDefinition(Definition[C], kind=True, format=3): """ -_UNASKED: Final[Mapping[CodecKind, tuple[str, ...]]] = { - "array_array": (), - "array_bytes": ("transition",), - "bytes_bytes": ("chunk_rules", "transition", "pipelines"), -} -"""The functions that are never called for a codec of each kind. A bytes -> bytes codec receives bytes, and only an array -> array codec passes on a chunk.""" - - @dataclass(frozen=True, kw_only=True, slots=True, repr=False) -class CodecDefinition(Definition[C], kind=True, format=3): +class CodecDefinition(Definition[C], ABC, kind=True, format=3): """A codec: what it converts its input to, and whether the size of its output is static. - A codec that receives an array (array -> array, array -> bytes) - declares the spec violations for a given input `Chunk` with - `chunk_rules`. It receives the configuration, the nested declarations as the - scope read them, and the chunk, and it locates its problems in the - configuration. For example, a `bytes` codec without `endian` that - receives a multi-byte data type is a problem. An array -> array codec - also declares its output with `transition`: given its input chunk, it - returns the chunk the next codec receives. For example, `transpose` - permutes the axes, and `cast_value` changes the data type. These two - functions correspond to the spec: a codec computes its output from the - shape and data type of its input, and "If the - decoded_representation_type is not supported, this algorithm must - fail with an error" - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L987-L994). - The transition is called for every input chunk, whatever the chunk - rules found, so it must return only what is true in either case. For - example, a `transpose` whose `order` has the wrong number of axes - passes on unknown lengths. A codec that does not declare its output - passes the next codec a chunk with nothing known about it. - - A codec that contains its own pipelines declares what each one - receives with `pipelines`. Given the codec's input chunk, it returns - the chunk that the first codec of each pipeline receives, keyed by the - configuration member that holds the pipeline. For example, a shard's - inner codecs receive its inner chunks, and its index codecs receive - the shard index. Like the transition, it is called whatever the chunk - rules found, and must return only what is true in either case. A - function that is never called for a codec of its kind is rejected, - such as the chunk rules of a bytes -> bytes codec, which receives - bytes. + A codec is one of three kinds, each its own class: + `ArrayArrayCodecDefinition`, `ArrayBytesCodecDefinition` and + `BytesBytesCodecDefinition`. The kind fixes where the codec may sit in + a pipeline, and what it is asked: a codec that receives an array, an + `ArrayCodecDefinition`, is asked about the chunk it receives, and a + bytes -> bytes codec, which receives bytes, is asked nothing about + them. This class holds what every codec declares, and is not built + itself. """ label: ClassVar[str] = "codec" @@ -356,39 +359,22 @@ class CodecDefinition(Definition[C], kind=True, format=3): codecs of static size. """ - kind: CodecKind - """What the codec does to what it is handed: array -> array, array -> bytes or bytes -> bytes. - - It fixes where the codec may sit in a pipeline, and which of `chunk_rules`, - `transition` and `pipelines` it may be given: building a definition with - one its kind is never asked raises `TypeError`. - """ size: CodecSize """Whether the size of what the codec gives out is fixed by the size of what it is handed. Where a member takes only codecs of static size, a codec of dynamic size is a problem. """ - chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules - """The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec.""" - transition: Callable[[C, Nested, Chunk], Chunk] = unknown_chunk - """The chunk the next codec receives, given the chunk this array -> array codec receives.""" - pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines - """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" + + @property + @abstractmethod + def kind(self) -> CodecKind: + """What the codec does to what it is handed: array -> array, array -> bytes or bytes -> bytes. It fixes where the codec may sit in a pipeline.""" def _refusal(self) -> str | None: - kind: object = self.kind - if kind not in get_args(CodecKind): - return f"{self.name!r}: kind is one of {get_args(CodecKind)!r}, got {kind!r}" size: object = self.size if size not in get_args(CodecSize): return f"{self.name!r}: size is one of {get_args(CodecSize)!r}, got {size!r}" - defaults = { - attribute.name: attribute.default for attribute in dataclasses.fields(CodecDefinition) - } - for attribute in _UNASKED[self.kind]: - if getattr(self, attribute) is not defaults[attribute]: - return f"{self.name!r}: {attribute}, which no codec of kind {kind!r} is asked" return None @classmethod @@ -408,6 +394,78 @@ def _alias_problems( ) +@dataclass(frozen=True, kw_only=True, slots=True, repr=False) +class ArrayCodecDefinition(CodecDefinition[C]): + """A codec that receives an array: an array -> array or an array -> bytes codec. + + It declares the spec violations for a given input `Chunk` with + `chunk_rules`. They receive the configuration, the nested declarations + as the scope read them, and the chunk, and they locate their problems + in the configuration. For example, a `bytes` codec without `endian` + that receives a multi-byte data type is a problem. This corresponds to + the spec: a codec computes its output from the shape and data type of + its input, and "If the decoded_representation_type is not supported, + this algorithm must fail with an error" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L987-L994). + + A codec that contains its own pipelines declares what each one + receives with `pipelines`. Given the codec's input chunk, it returns + the chunk that the first codec of each pipeline receives, keyed by the + configuration member that holds the pipeline. For example, a shard's + inner codecs receive its inner chunks, and its index codecs receive + the shard index. It is called whatever the chunk rules found, and must + return only what is true in either case. + """ + + chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules + """The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec.""" + pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines + """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" + + +@dataclass(frozen=True, kw_only=True, slots=True, repr=False) +class ArrayArrayCodecDefinition(ArrayCodecDefinition[C]): + """An array -> array codec, such as `transpose`. + + It declares its output with `transition`: given its input chunk, it + returns the chunk the next codec receives. For example, `transpose` + permutes the axes, and `cast_value` changes the data type. The + transition is called for every input chunk, whatever the chunk rules + found, so it must return only what is true in either case. For + example, a `transpose` whose `order` has the wrong number of axes + passes on unknown lengths. A codec that does not declare its output + passes the next codec a chunk with nothing known about it. + """ + + transition: Callable[[C, Nested, Chunk], Chunk] = unknown_chunk + """The chunk the next codec receives, given the chunk this codec receives.""" + + @property + def kind(self) -> CodecKind: + """`"array_array"`: the codec sits before the array -> bytes codec, and hands an array on.""" + return "array_array" + + +@dataclass(frozen=True, kw_only=True, slots=True, repr=False) +class ArrayBytesCodecDefinition(ArrayCodecDefinition[C]): + """An array -> bytes codec, such as `bytes`: the one codec of a pipeline that turns the array into bytes.""" + + @property + def kind(self) -> CodecKind: + """`"array_bytes"`: the codec turns the array into bytes, exactly once in a pipeline.""" + return "array_bytes" + + +@dataclass(frozen=True, kw_only=True, slots=True, repr=False) +class BytesBytesCodecDefinition(CodecDefinition[C]): + """A bytes -> bytes codec, such as `gzip`. It receives bytes, which have no shape or data type, so it is asked nothing about them.""" + + @property + def kind(self) -> CodecKind: + """`"bytes_bytes"`: the codec sits after the array -> bytes codec.""" + return "bytes_bytes" + + @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class StorageTransformerDefinition(Definition[C], kind=True, format=3): """A storage transformer.""" @@ -796,7 +854,11 @@ def chunk_grid_lengths( "RAW_BYTES_NAME", "RAW_BYTES_NAME_PATTERN", "STATIC_SIZE_ALIASES", + "ArrayArrayCodecDefinition", + "ArrayBytesCodecDefinition", + "ArrayCodecDefinition", "ArrayExtraFieldDefinition", + "BytesBytesCodecDefinition", "Chunk", "ChunkGridDefinition", "ChunkKeyEncodingDefinition", @@ -807,6 +869,7 @@ def chunk_grid_lengths( "ExtraFieldDefinition", "GroupExtraFieldDefinition", "Lengths", + "NumberClass", "StorageClass", "StorageTransformerDefinition", "chunk_grid_lengths", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index a46883d439..592bb18a5f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -43,6 +43,8 @@ ruled, ) from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, + ArrayCodecDefinition, Chunk, CodecDefinition, CodecKind, @@ -131,7 +133,8 @@ def read_pipeline( stages.append(Stage(codec, handed)) handed = None continue - if definition.kind == "bytes_bytes": + if not isinstance(definition, ArrayCodecDefinition): + # A bytes -> bytes codec receives bytes, which are not a chunk. stages.append(Stage(codec, None)) handed = None continue @@ -145,7 +148,7 @@ def read_pipeline( inner, found = _inner_pipelines(definition, configuration, nested, incoming, at) problems.extend(found) stages.append(Stage(codec, incoming, inner)) - if definition.kind == "array_bytes": + if not isinstance(definition, ArrayArrayCodecDefinition): handed = None elif not isinstance(codec, AcceptedDeclaration): handed = Chunk() @@ -202,7 +205,7 @@ def _order_problems( def _chunk_problems( - definition: CodecDefinition[Any], + definition: ArrayCodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, chunk: Chunk, @@ -235,7 +238,7 @@ def _codec_problems( def _inner_pipelines( - definition: CodecDefinition[Any], + definition: ArrayCodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, chunk: Chunk, @@ -299,7 +302,7 @@ def _held( def _handed_on( - definition: CodecDefinition[Any], + definition: ArrayArrayCodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, chunk: Chunk, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py index f56da6052e..60c473d48d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py @@ -1,85 +1,34 @@ -"""The data types that the arithmetic codecs, `cast_value` and `scale_offset`, accept. +"""The number classes that the arithmetic codecs, `cast_value` and `scale_offset`, refuse. `cast_value` "is only defined for data types that model real numbers: floating-point and integral data types" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L5-L9), and `scale_offset` "for data types where multiplication, division, addition, and subtraction are well-defined" -(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9); +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9). Each spec then lists "the following data types defined in this -repository", and the list is the same for both. A data type defined -elsewhere may also qualify, so a data type that the list does not name -is rejected only when the spec text itself excludes it. - -Neither codec rejects the numpy time types. Each of them "is compatible -with any codec that supports arrays of signed 64-bit integers" -(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120), -and both codecs support those, even though neither list names the time -types. +repository" that it supports, and the list is the same for both: the +integer, unsigned integer and floating-point types. That repository also +defines the complex types and the numpy time types, and neither list +names them. So both codecs accept the same data types. + +Each data type says what kind of numbers its values are with its +`number_class`, and both codecs refuse the classes their lists leave out. +A data type that does not declare its number class is accepted, since +nothing is known of its values. """ from typing import Final -from zarr_metadata.v3._definition import RAW_BYTES_NAME -from zarr_metadata.v3._types.data_type import ( - BOOL_DATA_TYPE_NAME, - BYTES_DATA_TYPE_NAME, - COMPLEX64_DATA_TYPE_NAME, - COMPLEX128_DATA_TYPE_NAME, - FIXED_LENGTH_UTF32_DATA_TYPE_NAME, - NULL_TERMINATED_BYTES_DATA_TYPE_NAME, - STRING_DATA_TYPE_NAME, - STRUCT_DATA_TYPE_NAME, - VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME, - ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME, -) - -FLOATING_POINT: Final = frozenset( - { - "float4_e2m1fn", - "float6_e2m3fn", - "float6_e3m2fn", - "float8_e3m4", - "float8_e4m3", - "float8_e4m3b11fnuz", - "float8_e4m3fnuz", - "float8_e5m2", - "float8_e5m2fnuz", - "float8_e8m0fnu", - "bfloat16", - "float16", - "float32", - "float64", - } -) -"""The floating-point data types that the lists name. None of them is integral, so `cast_value` may not wrap to them.""" - -NOT_NUMBERS: Final = frozenset( - { - BOOL_DATA_TYPE_NAME, - RAW_BYTES_NAME, - STRING_DATA_TYPE_NAME, - BYTES_DATA_TYPE_NAME, - STRUCT_DATA_TYPE_NAME, - FIXED_LENGTH_UTF32_DATA_TYPE_NAME, - NULL_TERMINATED_BYTES_DATA_TYPE_NAME, - VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME, - ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME, - } -) -"""Data types whose values are not numbers (truth values, raw bits, text, byte strings, records). Neither codec accepts them.""" +from zarr_metadata.v3._definition import NumberClass -COMPLEX: Final = frozenset({COMPLEX64_DATA_TYPE_NAME, COMPLEX128_DATA_TYPE_NAME}) -"""Complex data types. They do not model real numbers, so `cast_value` does not accept them. +NO_REAL_NUMBERS: Final[frozenset[NumberClass]] = frozenset({"non_numeric", "complex"}) +"""The number classes that neither codec takes: values that are not plain numbers, and complex numbers. Addition, subtraction, multiplication and division are well-defined on -them, but the `scale_offset` spec does not name them in its list. So the -spec leaves open whether `scale_offset` accepts them. +complex numbers, but the `scale_offset` spec names none of the complex +types its repository defines among those it supports. """ -NO_REAL_NUMBERS: Final = NOT_NUMBERS | COMPLEX -"""The data types that model no real number, which `cast_value` takes neither from nor to.""" - - -__all__ = ["COMPLEX", "FLOATING_POINT", "NOT_NUMBERS", "NO_REAL_NUMBERS"] +__all__ = ["NO_REAL_NUMBERS"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 5aac8b054d..0ed7cc40a3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -9,7 +9,7 @@ from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested -from zarr_metadata.v3._definition import CodecDefinition +from zarr_metadata.v3._definition import BytesBytesCodecDefinition from zarr_metadata.v3._types.codec import ( BLOSC_CNAME, BLOSC_CODEC_NAME, @@ -62,10 +62,9 @@ def _canonical(configuration: BloscCodecConfiguration) -> BloscCodecConfiguratio return simpler -BLOSC_CODEC: Final = CodecDefinition( +BLOSC_CODEC: Final = BytesBytesCodecDefinition( name=BLOSC_CODEC_NAME, configuration=BloscCodecConfiguration, - kind="bytes_bytes", size="dynamic", rules=_rules, canonical=_canonical, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 652c3d9f5f..5b08cf3b0a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -10,8 +10,8 @@ from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import ( + ArrayBytesCodecDefinition, Chunk, - CodecDefinition, storage_of, ) from zarr_metadata.v3._types.codec import ( @@ -64,10 +64,9 @@ def _chunk_rules( ) -BYTES_CODEC: Final = CodecDefinition( +BYTES_CODEC: Final = ArrayBytesCodecDefinition( name=BYTES_CODEC_NAME, configuration=BytesCodecConfiguration, - kind="array_bytes", size="static", chunk_rules=_chunk_rules, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index c00af8b8b1..377c92dfc1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -15,8 +15,9 @@ fill_value_problems, ) from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, Chunk, - CodecDefinition, + NumberClass, ) from zarr_metadata.v3._types.codec import ( CAST_OUT_OF_RANGE_MODE, @@ -31,7 +32,7 @@ ScalarMap, ScalarMapEntry, ) -from zarr_metadata.v3.codec._arithmetic import FLOATING_POINT, NO_REAL_NUMBERS +from zarr_metadata.v3.codec._arithmetic import NO_REAL_NUMBERS def _scalars( @@ -68,21 +69,25 @@ def _rules( (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L84); and a scalar of the target type is encoded with its fill value encoding. These rules check the target data type, which the scope - read. The chunk rules check the input data type. If the target does - not model real numbers, that is the only problem reported for it. + read, by the number class its definition declares. The chunk rules + check the input data type. If the target does not model real numbers, + that is the only problem reported for it. A target that does not + declare its number class is accepted, since nothing is known of its + values. """ target = nested.get(("data_type",)) if not isinstance(target, AcceptedDeclaration): return - name, written = target.definition.name, target.name - if name in NO_REAL_NUMBERS: + number_class: NumberClass | None = target.definition.number_class + written = target.name + if number_class in NO_REAL_NUMBERS: yield ValidationProblem( ("data_type",), f"expected a data type that models real numbers, got {shown(written)}", "invalid_value", ) return - if configuration.get("out_of_range") == "wrap" and name in FLOATING_POINT: + if configuration.get("out_of_range") == "wrap" and number_class == "floating_point": yield ValidationProblem( ("out_of_range",), f"expected an integral data_type to wrap to, got {shown(written)}", @@ -106,7 +111,7 @@ def _chunk_rules( source = chunk.data_type if not isinstance(source, AcceptedDeclaration): return - if source.definition.name in NO_REAL_NUMBERS: + if source.definition.number_class in NO_REAL_NUMBERS: yield ValidationProblem( (), f"expected a chunk of a data type that models real numbers, got {shown(source.name)}", @@ -129,10 +134,9 @@ def _transition(configuration: CastValueCodecConfiguration, nested: Nested, chun return Chunk(chunk.lengths, nested.get(("data_type",))) -CAST_VALUE_CODEC: Final = CodecDefinition( +CAST_VALUE_CODEC: Final = ArrayArrayCodecDefinition( name=CAST_VALUE_CODEC_NAME, configuration=CastValueCodecConfiguration, - kind="array_array", size="static", rules=_rules, chunk_rules=_chunk_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 0c94867d5a..bf1f2fe184 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -10,7 +10,7 @@ from typing import Final from zarr_metadata._common import EmptyConfiguration -from zarr_metadata.v3._definition import CodecDefinition +from zarr_metadata.v3._definition import BytesBytesCodecDefinition from zarr_metadata.v3._types.codec import ( CRC32C_CODEC_NAME, Crc32cCodecMetadata, @@ -18,8 +18,8 @@ Crc32cCodecObject, ) -CRC32C_CODEC: Final = CodecDefinition( - name=CRC32C_CODEC_NAME, configuration=EmptyConfiguration, kind="bytes_bytes", size="static" +CRC32C_CODEC: Final = BytesBytesCodecDefinition( + name=CRC32C_CODEC_NAME, configuration=EmptyConfiguration, size="static" ) """The `crc32c` codec. A checksum has no configuration.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 126e778797..1778e090cb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -6,7 +6,7 @@ from typing import Final -from zarr_metadata.v3._definition import CodecDefinition +from zarr_metadata.v3._definition import BytesBytesCodecDefinition from zarr_metadata.v3._types.codec import ( GZIP_CODEC_NAME, GzipCodecConfiguration, @@ -15,10 +15,9 @@ GzipCodecObject, ) -GZIP_CODEC: Final = CodecDefinition( +GZIP_CODEC: Final = BytesBytesCodecDefinition( name=GZIP_CODEC_NAME, configuration=GzipCodecConfiguration, - kind="bytes_bytes", size="dynamic", ) """The `gzip` codec.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 473eeb4c1f..459f953a0c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -14,8 +14,8 @@ fill_value_problems, ) from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, Chunk, - CodecDefinition, ) from zarr_metadata.v3._types.codec import ( SCALE_OFFSET_CODEC_NAME, @@ -24,7 +24,7 @@ ScaleOffsetCodecName, ScaleOffsetCodecObject, ) -from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS +from zarr_metadata.v3.codec._arithmetic import NO_REAL_NUMBERS def _rules( @@ -46,12 +46,15 @@ def _rules( def _chunk_rules( configuration: ScaleOffsetCodecConfiguration, nested: Nested, chunk: Chunk ) -> Iterator[ValidationProblem]: - """Check that the input data type supports arithmetic, and that `offset` and `scale` are values of that type. + """Check that the input data type is an integer or floating-point type, and that `offset` and `scale` are values of that type. The codec "is only defined for data types where multiplication, - division, addition, and subtraction are well-defined" + division, addition, and subtraction are well-defined", and of the data + types its repository defines it supports the integer, unsigned integer + and floating-point types (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9), - and each of `offset` and `scale` "MUST be encoded to JSON using the + so a data type that declares complex or non-numeric values is refused. + Each of `offset` and `scale` "MUST be encoded to JSON using the Zarr V3 fill value encoding for the input array's data type" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43). The rules, not the chunk rules, reject a null. @@ -59,10 +62,10 @@ def _chunk_rules( source = chunk.data_type if not isinstance(source, AcceptedDeclaration): return - if source.definition.name in NOT_NUMBERS: + if source.definition.number_class in NO_REAL_NUMBERS: yield ValidationProblem( (), - f"expected a chunk of a data type with arithmetic, got {shown(source.name)}", + f"expected a chunk of an integer or floating-point data type, got {shown(source.name)}", "invalid_value", ) return @@ -86,10 +89,9 @@ def _transition( return chunk -SCALE_OFFSET_CODEC: Final = CodecDefinition( +SCALE_OFFSET_CODEC: Final = ArrayArrayCodecDefinition( name=SCALE_OFFSET_CODEC_NAME, configuration=ScaleOffsetCodecConfiguration, - kind="array_array", size="static", rules=_rules, chunk_rules=_chunk_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 5b4b3556e8..5a4d0f5983 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -13,8 +13,8 @@ Nested, ) from zarr_metadata.v3._definition import ( + ArrayBytesCodecDefinition, Chunk, - CodecDefinition, Lengths, ) from zarr_metadata.v3._types.codec import ( @@ -134,11 +134,10 @@ def _canonical( return simpler -SHARDING_INDEXED_CODEC: Final = CodecDefinition( +SHARDING_INDEXED_CODEC: Final = ArrayBytesCodecDefinition( name=SHARDING_INDEXED_CODEC_NAME, configuration=ShardingIndexedCodecConfiguration, canonical=_canonical, - kind="array_bytes", size="dynamic", chunk_rules=_chunk_rules, pipelines=_pipelines, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index b4e34b2ab6..7d0adb9446 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -10,8 +10,8 @@ from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, Chunk, - CodecDefinition, ) from zarr_metadata.v3._types.codec import ( TRANSPOSE_CODEC_NAME, @@ -74,10 +74,9 @@ def _transition(configuration: TransposeCodecConfiguration, nested: Nested, chun return Chunk(tuple(lengths[axis] for axis in order), chunk.data_type) -TRANSPOSE_CODEC: Final = CodecDefinition( +TRANSPOSE_CODEC: Final = ArrayArrayCodecDefinition( name=TRANSPOSE_CODEC_NAME, configuration=TransposeCodecConfiguration, - kind="array_array", size="static", rules=_rules, chunk_rules=_chunk_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py index 8b2f42da68..a8ebdd230f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py @@ -14,7 +14,7 @@ from zarr_metadata._common import EmptyConfiguration from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested -from zarr_metadata.v3._definition import Chunk, CodecDefinition +from zarr_metadata.v3._definition import ArrayBytesCodecDefinition, Chunk from zarr_metadata.v3._types.codec import ( VLEN_BYTES_CODEC_NAME, VlenBytesCodecMetadata, @@ -42,10 +42,9 @@ def _chunk_rules( yield from vlen_problems(chunk, _COMPATIBLE) -VLEN_BYTES_CODEC: Final = CodecDefinition( +VLEN_BYTES_CODEC: Final = ArrayBytesCodecDefinition( name=VLEN_BYTES_CODEC_NAME, configuration=EmptyConfiguration, - kind="array_bytes", size="dynamic", chunk_rules=_chunk_rules, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py index a698288646..cb58373fbb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py @@ -14,7 +14,7 @@ from zarr_metadata._common import EmptyConfiguration from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested -from zarr_metadata.v3._definition import Chunk, CodecDefinition +from zarr_metadata.v3._definition import ArrayBytesCodecDefinition, Chunk from zarr_metadata.v3._types.codec import ( VLEN_UTF8_CODEC_NAME, VlenUtf8CodecMetadata, @@ -34,10 +34,9 @@ def _chunk_rules( yield from vlen_problems(chunk, _COMPATIBLE) -VLEN_UTF8_CODEC: Final = CodecDefinition( +VLEN_UTF8_CODEC: Final = ArrayBytesCodecDefinition( name=VLEN_UTF8_CODEC_NAME, configuration=EmptyConfiguration, - kind="array_bytes", size="dynamic", chunk_rules=_chunk_rules, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index a4153ca6f8..837f280921 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -8,7 +8,7 @@ from typing import Final -from zarr_metadata.v3._definition import CodecDefinition +from zarr_metadata.v3._definition import BytesBytesCodecDefinition from zarr_metadata.v3._types.codec import ( ZSTD_CODEC_NAME, ZSTD_MAX_LEVEL, @@ -34,11 +34,10 @@ def _canonical(configuration: ZstdCodecConfiguration) -> ZstdCodecConfiguration: return simpler -ZSTD_CODEC: Final = CodecDefinition( +ZSTD_CODEC: Final = BytesBytesCodecDefinition( name=ZSTD_CODEC_NAME, configuration=ZstdCodecConfiguration, canonical=_canonical, - kind="bytes_bytes", size="dynamic", ) """The `zstd` codec.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index df008b6710..fd309779d1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -20,6 +20,7 @@ fill_value=BoolFillValue, fill_value_default=ConstantFillValue(False), storage=single_byte, + number_class="non_numeric", ) """The `bool` data type. It is written as a bare name and has no configuration.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index 3aea0c2663..ca402ee052 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -52,6 +52,7 @@ def _fill_value_canonical( fill_value_canonical=_fill_value_canonical, storage=variable_length, array_bytes_codec_default=vlen_bytes_codec, + number_class="non_numeric", ) """The `bytes` data type. It is written as a bare name and has no configuration. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py index 40cb7d09df..43a70bd7d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py @@ -32,6 +32,7 @@ fill_value_rules=complex_fill_value_rules(FLOAT64_DATA_TYPE.fill_value_rules), fill_value_canonical=complex_fill_value_canonical(FLOAT64_DATA_TYPE.fill_value_canonical), storage=multi_byte, + number_class="complex", ) """The `complex128` data type. It is written as a bare name and has no configuration. Its fill value is a pair of `float64` components.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py index 55281b4973..e8ba97aab0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py @@ -32,6 +32,7 @@ fill_value_rules=complex_fill_value_rules(FLOAT32_DATA_TYPE.fill_value_rules), fill_value_canonical=complex_fill_value_canonical(FLOAT32_DATA_TYPE.fill_value_canonical), storage=multi_byte, + number_class="complex", ) """The `complex64` data type. It is written as a bare name and has no configuration. Its fill value is a pair of `float32` components.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py index b3e23251b7..7bb8c135e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py @@ -79,6 +79,7 @@ def _fill_value_canonical( fill_value_rules=_fill_value_rules, fill_value_canonical=_fill_value_canonical, storage=multi_byte, + number_class="non_numeric", ) """The `fixed_length_utf32` data type: NumPy's `U`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 4fe61579f8..27725e8925 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -47,6 +47,7 @@ def hex_float16(value: str) -> HexFloat16: fill_value_rules=float_fill_value_rules("float16", hex_float16), fill_value_canonical=float_fill_value_canonical(16), storage=multi_byte, + number_class="floating_point", ) """The `float16` data type. It is written as a bare name and has no configuration. Its fill value is a number, a named value or a hex string.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 9666121d80..d265b3d69f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -47,6 +47,7 @@ def hex_float32(value: str) -> HexFloat32: fill_value_rules=float_fill_value_rules("float32", hex_float32), fill_value_canonical=float_fill_value_canonical(32), storage=multi_byte, + number_class="floating_point", ) """The `float32` data type. It is written as a bare name and has no configuration. Its fill value is a number, a named value or a hex string.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index 69ccab10fc..db50d6a14f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -47,6 +47,7 @@ def hex_float64(value: str) -> HexFloat64: fill_value_rules=float_fill_value_rules("float64", hex_float64), fill_value_canonical=float_fill_value_canonical(64), storage=multi_byte, + number_class="floating_point", ) """The `float64` data type. It is written as a bare name and has no configuration. Its fill value is a number, a named value or a hex string.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py index 1fde31deb6..5788f40a2c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py @@ -24,6 +24,7 @@ fill_value=Int16FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `int16` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py index 496215175b..3da6dc94b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py @@ -24,6 +24,7 @@ fill_value=Int32FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `int32` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py index f0cf13d87a..136258c74b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py @@ -24,6 +24,7 @@ fill_value=Int64FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `int64` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py index 41f1a16e9c..809b72568f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py @@ -20,6 +20,7 @@ fill_value=Int8FillValue, fill_value_default=ConstantFillValue(0), storage=single_byte, + number_class="integral", ) """The `int8` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py index c5308a44ef..333f362cb5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py @@ -81,6 +81,7 @@ def _fill_value_canonical( fill_value_rules=_fill_value_rules, fill_value_canonical=_fill_value_canonical, storage=single_byte, + number_class="non_numeric", ) """zarr-python's `null_terminated_bytes` data type: NumPy's `S`. Each value is `length_bytes` single bytes.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 8ce3418c43..ea33f7463b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -29,8 +29,20 @@ fill_value_default=ConstantFillValue("NaT"), fill_value_canonical=numpy_time_fill_value_canonical, storage=numpy_time_storage, + number_class="non_numeric", ) -"""The `numpy.datetime64` data type.""" +"""The `numpy.datetime64` data type. + +Its number class is non-numeric: its values are moments in time, and +neither `cast_value` nor `scale_offset` names it among "the following data +types defined in this repository" that it supports +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L5-L9, +https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9). +Its own spec says it "is compatible with any codec that supports arrays of +signed 64-bit integers" +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120), +which those two codecs' specs do not bear out. +""" __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index b78d0cafb9..4aaa3210d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -27,8 +27,14 @@ fill_value_default=ConstantFillValue("NaT"), fill_value_canonical=numpy_time_fill_value_canonical, storage=numpy_time_storage, + number_class="non_numeric", ) -"""The `numpy.timedelta64` data type.""" +"""The `numpy.timedelta64` data type. + +Its number class is non-numeric, as that of `numpy.datetime64` is: its +values are durations, and neither `cast_value` nor `scale_offset` names it +among the data types it supports. +""" __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 9cf03cc08b..63fc15e878 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -93,6 +93,7 @@ def _storage(configuration: RawBytesConfiguration, nested: Nested) -> StorageCla fill_value_default=_fill_value_default, fill_value_rules=_fill_value_rules, storage=_storage, + number_class="non_numeric", ) """Raw bits, `r*`: a single data type whose name carries its size. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index f7017d571c..f137cda8c5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -32,6 +32,7 @@ def _array_bytes_codec_default(*_: object) -> VlenUtf8CodecObject: fill_value_default=ConstantFillValue(""), storage=variable_length, array_bytes_codec_default=_array_bytes_codec_default, + number_class="non_numeric", ) """The `string` data type. It is written as a bare name and has no configuration. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index e7f1a54b27..af9fbc0038 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -164,6 +164,7 @@ def _storage(configuration: StructConfiguration, nested: Nested) -> StorageClass fill_value_rules=_fill_value_rules, fill_value_canonical=_fill_value_canonical, storage=_storage, + number_class="non_numeric", ) """The `struct` data type: a record of named fields. The type of each field is a nested declaration.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py index 808dd1792a..5129fabaa0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py @@ -24,6 +24,7 @@ fill_value=Uint16FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `uint16` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py index acda4e78b8..ede4821c21 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py @@ -24,6 +24,7 @@ fill_value=Uint32FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `uint32` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py index a6d72e473d..fa404617c2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py @@ -24,6 +24,7 @@ fill_value=Uint64FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `uint64` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py index 1aea655edf..339a48fe4d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py @@ -24,6 +24,7 @@ fill_value=Uint8FillValue, fill_value_default=ConstantFillValue(0), storage=single_byte, + number_class="integral", ) """The `uint8` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py index 0e4b241d9e..4715758909 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py @@ -43,6 +43,7 @@ def _fill_value_canonical( fill_value_canonical=_fill_value_canonical, storage=variable_length, array_bytes_codec_default=vlen_bytes_codec, + number_class="non_numeric", ) """zarr-python's `variable_length_bytes` data type. It is written as a bare name and has no configuration, and each value is stored in as many bytes as it needs.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py index eef9ed81f9..e1f41efcc9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py @@ -88,6 +88,7 @@ def _fill_value_default(configuration: ZarrPythonRawBytesConfiguration, nested: fill_value_rules=_fill_value_rules, fill_value_canonical=_fill_value_canonical, storage=single_byte, + number_class="non_numeric", ) """zarr-python's `raw_bytes` data type: NumPy's `V` without fields. Each value is `length_bytes` single bytes.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index 4b06e91a22..c98d6460c1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -1,9 +1,10 @@ """The kinds of metadata in a Zarr v3 document, and the scopes this package provides for it. The kind of metadata a definition defines is its type: `CodecDefinition` -(with the codec's `kind`, and its `size`: `"static"` if the output size -is determined by the input size, or `"dynamic"` if it depends on the -values), `DataTypeDefinition`, `ChunkGridDefinition`, +(built as one of its kinds, `ArrayArrayCodecDefinition`, +`ArrayBytesCodecDefinition` or `BytesBytesCodecDefinition`, with its +`size`: `"static"` if the output size is determined by the input size, +or `"dynamic"` if it depends on the values), `DataTypeDefinition`, `ChunkGridDefinition`, `ChunkKeyEncodingDefinition`, `StorageTransformerDefinition`, and, for the members of a document that the spec does not define, `ArrayExtraFieldDefinition` and `GroupExtraFieldDefinition`. What a @@ -53,6 +54,13 @@ `vlen-utf8` for `string`, and `bytes` with little `endian` for a data type that does not declare it. `default_array_bytes_codec(data_type)` calls it for a data type declaration the scope read. +A data type also declares what kind of numbers its values are with +`number_class`, a `NumberClass`: integral, floating-point, complex, or +not plain numbers. A codec that is defined only for some kinds of numbers +reads it: `cast_value` and `scale_offset` refuse a data type that +declares complex or non-numeric values, as the numpy time types do. A +data type that does not declare it leaves its number class unknown, and +those codecs accept it. A chunk grid declares which array shapes it fits with `shape_rules`, a function that yields the spec violations in a grid with its @@ -68,8 +76,8 @@ A codec is checked against its input. The array passes its first codec a `Chunk`: the lengths of the grid's chunks along each of the array's dimensions, and the array's data type declaration, with None for unknown -parts. A codec that receives an array declares the spec violations for -a given chunk with `chunk_rules`, located in its configuration (for +parts. A codec that receives an array, an `ArrayCodecDefinition`, +declares the spec violations for a given chunk with `chunk_rules`, located in its configuration (for example, a `transpose` whose `order` has the wrong number of axes). An array -> array codec declares what it passes to the next codec with `transition`, whatever its chunk rules found. For example, `transpose` @@ -118,7 +126,11 @@ ZARR_PYTHON_CONTEXT, ) from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, + ArrayBytesCodecDefinition, + ArrayCodecDefinition, ArrayExtraFieldDefinition, + BytesBytesCodecDefinition, Chunk, ChunkGridDefinition, ChunkKeyEncodingDefinition, @@ -129,6 +141,7 @@ ExtraFieldDefinition, GroupExtraFieldDefinition, Lengths, + NumberClass, StorageClass, StorageTransformerDefinition, default_array_bytes_codec, @@ -140,7 +153,11 @@ "CORE_AND_EXTENSIONS_CONTEXT", "CORE_CONTEXT", "ZARR_PYTHON_CONTEXT", + "ArrayArrayCodecDefinition", + "ArrayBytesCodecDefinition", + "ArrayCodecDefinition", "ArrayExtraFieldDefinition", + "BytesBytesCodecDefinition", "Chunk", "ChunkGridDefinition", "ChunkKeyEncodingDefinition", @@ -151,6 +168,7 @@ "ExtraFieldDefinition", "GroupExtraFieldDefinition", "Lengths", + "NumberClass", "Stage", "StorageClass", "StorageTransformerDefinition", diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 65cccd3dde..7fd62c5442 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -61,6 +61,7 @@ CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, ZARR_PYTHON_CONTEXT, + ArrayBytesCodecDefinition, ArrayExtraFieldDefinition, CodecDefinition, DataTypeDefinition, @@ -388,10 +389,9 @@ def _acme_vlen_text(configuration: EmptyConfiguration, nested: Nested) -> "ZarrV ), ], codecs=[ - CodecDefinition( + ArrayBytesCodecDefinition( name="acme.vlen-text", configuration=EmptyConfiguration, - kind="array_bytes", size="dynamic", ) ], diff --git a/packages/zarr-metadata/tests/model/test_consolidating.py b/packages/zarr-metadata/tests/model/test_consolidating.py index 8e5d5b505f..04e60cc1a4 100644 --- a/packages/zarr-metadata/tests/model/test_consolidating.py +++ b/packages/zarr-metadata/tests/model/test_consolidating.py @@ -25,7 +25,7 @@ from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, - CodecDefinition, + ArrayBytesCodecDefinition, ZarrV3Context, ) @@ -38,9 +38,7 @@ LOOSE_ZSTD = {"name": "zstd", "configuration": {"level": 30}} WITH_ZSTD = {**ARRAY, "codecs": (*ARRAY["codecs"], ZSTD)} WITH_LOOSE_ZSTD = {**ARRAY, "codecs": (*ARRAY["codecs"], LOOSE_ZSTD)} -MY_BYTES = CodecDefinition( - name="bytes", configuration=EmptyConfiguration, kind="array_bytes", size="static" -) +MY_BYTES = ArrayBytesCodecDefinition(name="bytes", configuration=EmptyConfiguration, size="static") PRIVATE = CORE_CONTEXT.copy(update={"codecs": {**CORE_CONTEXT.codecs, "bytes": MY_BYTES}}) INLINE: dict[str, Any] = {"kind": "inline", "must_understand": False} @@ -135,8 +133,8 @@ def test_error_a_model_that_conflicts_with_the_scope_is_refused_at_its_path() -> assert problem.kind == "invalid_value" assert problem.message == ( "expected a document read in the scope of the consolidated metadata, got a model that " - "reads the codec 'bytes' by CodecDefinition(name='bytes') of EmptyConfiguration, which " - "that scope reads by CodecDefinition(name='bytes') of BytesCodecConfiguration" + "reads the codec 'bytes' by ArrayBytesCodecDefinition(name='bytes') of EmptyConfiguration, which " + "that scope reads by ArrayBytesCodecDefinition(name='bytes') of BytesCodecConfiguration" ) @@ -341,7 +339,7 @@ def test_error_a_conflict_between_definitions_alike_says_they_differ() -> None: assert problem.message == ( "expected a document read in the scope of the consolidated metadata, got a model that " "reads the codec 'gzip' by another definition than the one that scope reads it by, " - "CodecDefinition(name='gzip') of GzipCodecConfiguration" + "BytesBytesCodecDefinition(name='gzip') of GzipCodecConfiguration" ) diff --git a/packages/zarr-metadata/tests/model/test_extension_points.py b/packages/zarr-metadata/tests/model/test_extension_points.py index 14b58999a3..e7c9fecc98 100644 --- a/packages/zarr-metadata/tests/model/test_extension_points.py +++ b/packages/zarr-metadata/tests/model/test_extension_points.py @@ -27,7 +27,7 @@ from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, - CodecDefinition, + BytesBytesCodecDefinition, ) if TYPE_CHECKING: @@ -48,10 +48,9 @@ class LenientGzipConfiguration(TypedDict, closed=True): level: int -LENIENT_GZIP = CodecDefinition( +LENIENT_GZIP = BytesBytesCodecDefinition( name="gzip", configuration=LenientGzipConfiguration, - kind="bytes_bytes", size="dynamic", ) """A reader's own gzip definition, which accepts any level. A scope can be extended, and an extension can replace a definition.""" diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index c60d1e3618..a0956e93c0 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -51,6 +51,7 @@ from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, + BytesBytesCodecDefinition, CodecDefinition, ZarrV3Context, ) @@ -642,9 +643,7 @@ def test_error_a_consolidated_envelope_reports_the_members_it_lacks_in_its_order ] -ACME_X = CodecDefinition( - name="acme.x", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" -) +ACME_X = BytesBytesCodecDefinition(name="acme.x", configuration=EmptyConfiguration, size="dynamic") def test_group_replace_keeps_the_documents_it_holds() -> None: diff --git a/packages/zarr-metadata/tests/model/test_pair.py b/packages/zarr-metadata/tests/model/test_pair.py index 5800762e2f..998a45e3f2 100644 --- a/packages/zarr-metadata/tests/model/test_pair.py +++ b/packages/zarr-metadata/tests/model/test_pair.py @@ -36,7 +36,8 @@ from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, - CodecDefinition, + ArrayBytesCodecDefinition, + BytesBytesCodecDefinition, ZarrV3Context, ) @@ -53,9 +54,7 @@ "chunk_key_encoding": {"name": "default"}, "codecs": ["bytes"], } -MY_BYTES = CodecDefinition( - name="bytes", configuration=EmptyConfiguration, kind="array_bytes", size="static" -) +MY_BYTES = ArrayBytesCodecDefinition(name="bytes", configuration=EmptyConfiguration, size="static") """A private `bytes` definition: a different meaning under the core name, with no configuration.""" PRIVATE = CORE_CONTEXT.copy(update={"codecs": {**CORE_CONTEXT.codecs, "bytes": MY_BYTES}}) SPELLED_OUT: dict[str, Any] = { @@ -178,10 +177,9 @@ def test_a_model_round_trips_through_its_document_in_its_scope( def test_error_a_model_whose_scope_does_not_pickle_says_so() -> None: """A scope that holds a definition with a local function cannot be pickled. Pickling the model raises the same error as pickling the `Context`.""" - local = CodecDefinition( + local = BytesBytesCodecDefinition( name="acme.c", configuration=EmptyConfiguration, - kind="bytes_bytes", size="dynamic", rules=lambda configuration, nested: iter(()), ) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 926b7753d4..e09dc0a0a8 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -325,6 +325,8 @@ def test_all_is_grouped_and_unique() -> None: "Loc", "Nested", "NodeName", + # What kind of numbers a data type's values are, as `StorageClass` says how they are stored. + "NumberClass", "NodePath", # The result of reading a declaration in a scope: `AcceptedDeclaration` (read by the # definition that claims its name), `UnclaimedDeclaration`, or `RefusedDeclaration`. diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 15800e471a..8ef6023265 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -22,6 +22,7 @@ import pytest from hypothesis import event, find, given, settings from hypothesis import strategies as st +from typing_extensions import TypedDict from tests.testing._shares import share_of from zarr_metadata import UNSET @@ -80,6 +81,9 @@ from zarr_metadata.v3.definition import ( CORE_CONTEXT, ZARR_PYTHON_CONTEXT, + ArrayBytesCodecDefinition, + ArrayCodecDefinition, + BytesBytesCodecDefinition, Chunk, ChunkGridDefinition, CodecDefinition, @@ -236,6 +240,8 @@ def _chunks(draw: st.DrawFn) -> Chunk: def _handed_problems(codec: Any, chunk: Chunk) -> list[object]: # noqa: ANN401 """What is wrong with `codec`, a codec field read, handed `chunk`: by its chunk rules, and in each pipeline it holds.""" definition = codec.definition + if not isinstance(definition, ArrayCodecDefinition): + return [] # a bytes -> bytes codec is handed bytes, not the chunk problems: list[object] = list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) handed = definition.pipelines(codec.configuration, codec.nested, chunk) for member, inner in handed.items(): @@ -245,13 +251,15 @@ def _handed_problems(codec: Any, chunk: Chunk) -> list[object]: # noqa: ANN401 def _holds_pipelines(definition: CodecDefinition[Any]) -> bool: - return definition.pipelines is not no_pipelines + return isinstance(definition, ArrayCodecDefinition) and definition.pipelines is not no_pipelines def _encoders_refuse(data: st.DataObject, chunk: Chunk) -> bool: """Whether every array -> bytes codec that holds no pipelines refuses a chunk of `chunk`'s data type.""" of_data_type = Chunk(None, chunk.data_type) - encoders = [d for d in _CODECS if d.kind == "array_bytes" and not _holds_pipelines(d)] + encoders = [ + d for d in _CODECS if isinstance(d, ArrayBytesCodecDefinition) and not _holds_pipelines(d) + ] for encoder in encoders: codec = _read(encoder, data.draw(declarations(encoder))) if list(encoder.chunk_rules(codec.configuration, codec.nested, of_data_type)) == []: @@ -277,6 +285,7 @@ def test_codecs_take_their_chunk( if _holds_pipelines(definition): assert _encoders_refuse(data, chunk) return + assert isinstance(definition, ArrayCodecDefinition) codec = _read(definition, data.draw(declarations(definition))) assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) != [] return @@ -515,6 +524,18 @@ def test_fill_values_of_a_field_no_definition_reads_is_a_lookup_error() -> None: fill_values(STRUCT_DATA_TYPE, configuration, strategies=without_float32) +class _AcmeIntConfiguration(TypedDict, closed=True): + """The configuration of `acme.int`: the member a refused draw leaves out.""" + + bits: int + + +_ACME_INT = DataTypeDefinition( + name="acme.int", configuration=_AcmeIntConfiguration, number_class="integral" +) +"""An integer type with a member it requires, which `cast_value` casts to.""" + + def test_a_refused_nested_draw_is_a_value_error() -> None: """A field a configuration holds is read as it is drawn, so a strategy that draws a refused one is caught there.""" refused = DataTypeStrategy( @@ -523,7 +544,7 @@ def test_a_refused_nested_draw_is_a_value_error() -> None: ) strategies: Strategies = { CAST_VALUE_CODEC: DEFAULT_STRATEGIES[CAST_VALUE_CODEC], - NUMPY_DATETIME64_DATA_TYPE: refused, + _ACME_INT: refused, } with pytest.raises(ValueError, match="read in the strategies. scope, is refused"): find( @@ -618,56 +639,25 @@ def test_scale_offset_reaches_every_spelling_of_a_float( ) -@pytest.mark.parametrize("definition", [COMPLEX64_DATA_TYPE, COMPLEX128_DATA_TYPE], ids=_id) -@pytest.mark.parametrize("member", ["offset", "scale"]) -@pytest.mark.parametrize("part", [0, 1]) -@pytest.mark.parametrize("value", ["NaN", "Infinity", "-Infinity"]) -def test_scale_offset_reaches_a_non_finite_part_of_a_complex( - definition: DataTypeDefinition[Any], member: str, part: int, value: str -) -> None: - """A complex number's `offset` and `scale` are pairs of float fill values, either of them non-finite.""" - chunk = Chunk(None, _read(definition, definition.name)) - configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) - assert configurations is not None - find( - configurations, - lambda configuration: ( - member in configuration - and cast("tuple[object, object]", configuration[member])[part] == value - ), - settings=_REACH, - ) - - def _is_zero_spelled(value: object, width: int) -> bool: """Whether a float fill value is zero of either sign, by its value alone.""" bits = _bits(value) return value == 0 if bits is None else bits in (0, 2 ** (width - 1)) -@pytest.mark.parametrize( - ("definition", "width"), - [ - *_FLOAT_BITS.items(), - (COMPLEX64_DATA_TYPE, 32), - (COMPLEX128_DATA_TYPE, 64), - ], - ids=str, -) +@pytest.mark.parametrize(("definition", "width"), list(_FLOAT_BITS.items()), ids=str) @given(data=st.data()) def test_scale_offset_scale_is_never_zero( definition: DataTypeDefinition[Any], width: int, data: st.DataObject ) -> None: - """No spelling of a float, nor of both parts of a complex number, draws a scale of zero: decoding divides by it.""" + """No spelling of a float draws a scale of zero: decoding divides by it.""" chunk = Chunk(None, _read(definition, definition.name)) configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) assert configurations is not None configuration = data.draw(configurations) assert list(SCALE_OFFSET_CODEC.chunk_rules(configuration, {}, chunk)) == [] if "scale" in configuration: - scale = configuration["scale"] - parts = scale if isinstance(scale, tuple) else (scale,) - assert not all(_is_zero_spelled(part, width) for part in parts) + assert not _is_zero_spelled(configuration["scale"], width) def _entries(dimension: object) -> tuple[object, ...]: @@ -932,11 +922,9 @@ def _read_data_type(field: object) -> Any: # noqa: ANN401 in { *(f"{sign}int{bits}" for sign in ("", "u") for bits in (8, 16, 32, 64)), *(f"float{bits}" for bits in (16, 32, 64)), - "numpy.datetime64", - "numpy.timedelta64", } ] -"""The data types `cast_value` casts to: integers and floats, and the numpy times, which are signed 64-bit integers.""" +"""The data types `cast_value` casts to: integers and floats.""" @pytest.mark.parametrize("definition", _REAL_NUMBERS, ids=_id) @@ -966,8 +954,8 @@ def test_cast_value_draws_only_real_number_data_types(declaration: ZarrV3Declara assert target.definition in _REAL_NUMBERS -_PLAIN_GZIP = CodecDefinition( - name=GZIP_CODEC.name, configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" +_PLAIN_GZIP = BytesBytesCodecDefinition( + name=GZIP_CODEC.name, configuration=EmptyConfiguration, size="dynamic" ) """A codec named as one the defaults hold, with a configuration of its own.""" @@ -1010,10 +998,9 @@ def test_a_later_definition_replaces_a_default_of_its_name(pipeline: tuple[Any, _PLAIN_STRUCT = DataTypeDefinition(name=STRUCT_DATA_TYPE.name, configuration=EmptyConfiguration) """A data type named as the struct the defaults hold, with a configuration of its own.""" -_PLAIN_SHARD = CodecDefinition( +_PLAIN_SHARD = ArrayBytesCodecDefinition( name=SHARDING_INDEXED_CODEC.name, configuration=EmptyConfiguration, - kind="array_bytes", size="dynamic", ) """A codec named as the shard the defaults hold, with a configuration of its own.""" diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index f261692fb8..d88dcf639e 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -57,6 +57,8 @@ from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, + ArrayArrayCodecDefinition, + BytesBytesCodecDefinition, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, @@ -100,10 +102,9 @@ def acme_stack_rules( ) -ACME_STACK = CodecDefinition( +ACME_STACK = BytesBytesCodecDefinition( name="acme.stack", configuration=AcmeStackConfiguration, - kind="bytes_bytes", size="dynamic", rules=acme_stack_rules, ) @@ -176,35 +177,32 @@ def acme_paired_rules( yield ValidationProblem((), "expected first and second, or neither", "invalid_value") -ACME_LEVEL = CodecDefinition( - name="acme.level", configuration=AcmeLevelConfiguration, kind="bytes_bytes", size="dynamic" +ACME_LEVEL = BytesBytesCodecDefinition( + name="acme.level", configuration=AcmeLevelConfiguration, size="dynamic" ) -ACME_TOTAL = CodecDefinition( - name="acme.total", configuration=AcmeTotalConfiguration, kind="bytes_bytes", size="dynamic" +ACME_TOTAL = BytesBytesCodecDefinition( + name="acme.total", configuration=AcmeTotalConfiguration, size="dynamic" ) -ACME_TREE = CodecDefinition( - name="acme.tree", configuration=AcmeTreeConfiguration, kind="bytes_bytes", size="dynamic" +ACME_TREE = BytesBytesCodecDefinition( + name="acme.tree", configuration=AcmeTreeConfiguration, size="dynamic" ) -ACME_FALLBACK = CodecDefinition( +ACME_FALLBACK = BytesBytesCodecDefinition( name="acme.fallback", configuration=AcmeFallbackConfiguration, - kind="bytes_bytes", size="dynamic", ) -ACME_ROUTES = CodecDefinition( - name="acme.routes", configuration=AcmeRoutesConfiguration, kind="bytes_bytes", size="dynamic" +ACME_ROUTES = BytesBytesCodecDefinition( + name="acme.routes", configuration=AcmeRoutesConfiguration, size="dynamic" ) -ACME_BOUNDED = CodecDefinition( +ACME_BOUNDED = BytesBytesCodecDefinition( name="acme.bounded", configuration=AcmeBoundedConfiguration, - kind="bytes_bytes", size="dynamic", rules=acme_bounded_rules, ) -ACME_PAIRED = CodecDefinition( +ACME_PAIRED = BytesBytesCodecDefinition( name="acme.paired", configuration=AcmePairedConfiguration, - kind="bytes_bytes", size="dynamic", rules=acme_paired_rules, ) @@ -373,8 +371,8 @@ class _HoldsV3Codecs(TypedDict, closed=True): ("build", "named"), [ ( - lambda: CodecDefinition( - name="acme.mix", configuration=_HoldsAV2DataType, kind="array_array", size="static" + lambda: ArrayArrayCodecDefinition( + name="acme.mix", configuration=_HoldsAV2DataType, size="static" ), "ZarrV2DataTypeDeclaration", ), @@ -383,16 +381,15 @@ class _HoldsV3Codecs(TypedDict, closed=True): "ZarrV3CodecDeclaration", ), ( - lambda: CodecDefinition( - name="acme.mix", configuration=_HoldsV2DataTypes, kind="array_array", size="static" + lambda: ArrayArrayCodecDefinition( + name="acme.mix", configuration=_HoldsV2DataTypes, size="static" ), "ZarrV2DataTypeDeclaration", ), ( - lambda: CodecDefinition( + lambda: ArrayArrayCodecDefinition( name="acme.mix", configuration=_HoldsANestedV2DataType, - kind="array_array", size="static", ), "ZarrV2DataTypeDeclaration", @@ -443,7 +440,8 @@ def test_error_a_declaration_read_by_hand_by_a_definition_filed_under_another_na def test_error_a_declaration_refused_by_hand_by_a_definition_of_another_kind() -> None: with pytest.raises( - TypeError, match="read as a DataTypeDefinition is read by one, got CodecDefinition" + TypeError, + match="read as a DataTypeDefinition is read by one, got BytesBytesCodecDefinition", ): RefusedDeclaration( json="gzip", name="gzip", read_as=DataTypeDefinition, definition=GZIP_CODEC @@ -1148,10 +1146,9 @@ def test_read_configuration_is_the_check_and_then_the_rules() -> None: def test_a_scope_takes_a_name_over() -> None: - strict = CodecDefinition( + strict = BytesBytesCodecDefinition( name="gzip", configuration=GzipCodecConfiguration, - kind="bytes_bytes", size="dynamic", rules=lambda configuration, nested: ( [ValidationProblem(("level",), "level 0 stores uncompressed", "invalid_value")] @@ -1335,7 +1332,7 @@ def test_a_declaration_copied_or_pickled_is_read_by_a_definition_equal_to_its_ow def test_a_definition_is_shown_by_its_kind_and_name() -> None: # Kept short, because a reading that holds the definition shows it this way. - assert repr(GZIP_CODEC) == "CodecDefinition(name='gzip')" + assert repr(GZIP_CODEC) == "BytesBytesCodecDefinition(name='gzip')" assert repr(INT8_DATA_TYPE) == "DataTypeDefinition(name='int8')" @@ -1366,29 +1363,26 @@ class AcmeCodecDefinition(CodecDefinition[Any]): def test_error_a_definition_configuration_is_a_typeddict() -> None: with pytest.raises(TypeError, match="give the TypedDict"): - CodecDefinition(name="acme.bad", configuration=dict, kind="bytes_bytes", size="dynamic") + BytesBytesCodecDefinition(name="acme.bad", configuration=dict, size="dynamic") def test_error_a_definition_member_is_a_shape_json_takes() -> None: with pytest.raises(TypeError, match="Unreadable: members is not a shape JSON takes"): - CodecDefinition( - name="acme.bad", configuration=Unreadable, kind="bytes_bytes", size="dynamic" - ) + BytesBytesCodecDefinition(name="acme.bad", configuration=Unreadable, size="dynamic") def test_error_a_configuration_says_what_its_other_keys_are() -> None: # An open TypedDict would accept a misspelled key without reporting it. with pytest.raises(TypeError, match="Loose says nothing of the keys it does not declare"): - CodecDefinition(name="acme.loose", configuration=Loose, kind="bytes_bytes", size="dynamic") + BytesBytesCodecDefinition(name="acme.loose", configuration=Loose, size="dynamic") def test_error_a_member_typed_as_plain_declaration_json_is_refused() -> None: # It would be checked only as JSON, and its name would never be matched to a definition. with pytest.raises(TypeError, match="annotate it with the declaration alias of its kind"): - CodecDefinition( + BytesBytesCodecDefinition( name="acme.plain", configuration=AcmePlainConfiguration, - kind="bytes_bytes", size="dynamic", ) @@ -1396,10 +1390,9 @@ def test_error_a_member_typed_as_plain_declaration_json_is_refused() -> None: def test_error_a_configuration_whose_annotations_do_not_resolve() -> None: # The error names the TypedDict that holds the annotation. with pytest.raises(TypeError, match="AcmeDecimal: name 'Decimal' is not defined"): - CodecDefinition( + BytesBytesCodecDefinition( name="acme.unresolved", configuration=AcmeUnresolvedConfiguration, - kind="bytes_bytes", size="dynamic", ) @@ -1417,10 +1410,9 @@ def test_error_a_configuration_with_a_constraint_the_checker_does_not_read() -> r".*; check it in the definition's `rules`" ), ): - CodecDefinition( + BytesBytesCodecDefinition( name="acme.unchecked", configuration=AcmeUncheckedConfiguration, - kind="bytes_bytes", size="dynamic", ) @@ -1434,21 +1426,17 @@ def test_error_a_fill_value_with_a_constraint_its_type_cannot_take() -> None: def test_error_a_definition_name_is_a_string() -> None: with pytest.raises(TypeError, match="a definition's name is a string"): - CodecDefinition( + BytesBytesCodecDefinition( name=5, # pyright: ignore[reportArgumentType] configuration=EmptyConfiguration, - kind="bytes_bytes", size="dynamic", ) -def test_error_a_codec_kind_is_one_of_three() -> None: - with pytest.raises(TypeError, match="kind is one of"): - CodecDefinition( - name="acme.k", - configuration=EmptyConfiguration, - kind="bytes_to_array", # pyright: ignore[reportArgumentType] - size="dynamic", +def test_error_a_codec_is_built_as_one_of_its_kinds() -> None: + with pytest.raises(TypeError, match="abstract class CodecDefinition"): + CodecDefinition( # pyright: ignore[reportAbstractUsage] + name="acme.k", configuration=EmptyConfiguration, size="dynamic" ) @@ -1484,16 +1472,16 @@ def test_error_a_data_type_fill_value_holding_a_declaration() -> None: (DataTypeDefinition, "storage"), (ChunkGridDefinition, "shape_rules"), (ChunkGridDefinition, "chunk_lengths"), - (CodecDefinition, "chunk_rules"), - (CodecDefinition, "transition"), - (CodecDefinition, "pipelines"), + (ArrayArrayCodecDefinition, "chunk_rules"), + (ArrayArrayCodecDefinition, "transition"), + (ArrayArrayCodecDefinition, "pipelines"), ], ) def test_error_a_function_member_that_is_not_a_function( kind: type[Definition[Any]], member: str ) -> None: # Each member that the definition's annotations declare as a `Callable`. - codec = {"kind": "array_array", "size": "static"} if kind is CodecDefinition else {} + codec = {"size": "static"} if issubclass(kind, CodecDefinition) else {} with pytest.raises(TypeError, match=f"'acme.t': {member} is a function, got 'none'"): kind(name="acme.t", configuration=EmptyConfiguration, **codec, **{member: "none"}) # pyright: ignore[reportArgumentType] @@ -1575,9 +1563,7 @@ def test_error_a_bad_name_is_a_problem_when_the_declaration_is_not_json_too() -> def test_error_a_definition_is_named_as_the_spec_names_an_extension(name: str) -> None: # No document could use this name, so the definition would never read anything. with pytest.raises(TypeError, match="so no document names it"): - CodecDefinition( - name=name, configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" - ) + BytesBytesCodecDefinition(name=name, configuration=EmptyConfiguration, size="dynamic") def test_error_a_declaration_built_by_hand_is_named_as_the_spec_names_one() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_kinds.py b/packages/zarr-metadata/tests/v3/test_kinds.py index 1939347603..2638e49e1c 100644 --- a/packages/zarr-metadata/tests/v3/test_kinds.py +++ b/packages/zarr-metadata/tests/v3/test_kinds.py @@ -20,6 +20,7 @@ from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context, ZarrV2DataTypeDefinition from zarr_metadata.v3.definition import ( CORE_CONTEXT, + BytesBytesCodecDefinition, CodecDefinition, DataTypeDefinition, ZarrV3Context, @@ -27,7 +28,7 @@ @dataclass(frozen=True, kw_only=True, slots=True, repr=False) -class MyCodec(CodecDefinition[Any]): +class MyCodec(BytesBytesCodecDefinition[Any]): """A codec definition with an extra attribute of its own. It is still a codec.""" note: str = "" @@ -40,7 +41,7 @@ class NoKind(Definition[Any]): def test_a_subclass_of_a_kind_is_a_definition_of_that_kind() -> None: """A definition built as a subclass of `CodecDefinition` is filed, read and named as a codec: the kind is the nearest class in the MRO declared with `kind=True`, not the class of the definition itself.""" - mine = MyCodec(name="mine", configuration=EmptyConfiguration, kind="bytes_bytes", size="static") + mine = MyCodec(name="mine", configuration=EmptyConfiguration, size="static") assert kind_of(mine) is CodecDefinition context = ZarrV3Context(codecs=[mine]) assert context.claimant(CodecDefinition, "mine") is mine diff --git a/packages/zarr-metadata/tests/v3/test_number_class.py b/packages/zarr-metadata/tests/v3/test_number_class.py new file mode 100644 index 0000000000..6eb3ba4e1a --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_number_class.py @@ -0,0 +1,162 @@ +"""Test the number class a data type declares, and the arithmetic codecs that read it. + +`cast_value` accepts only data types that model real numbers, on either +side of the cast, and wraps only to an integral one. `scale_offset` +accepts the same data types. Each reads the number class that +the data type's definition declares, and accepts a data type that +declares none, since nothing is known of its values. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, get_args + +import pytest + +from zarr_metadata.definition import EmptyConfiguration, resolve +from zarr_metadata.v3._pipeline import read_pipeline +from zarr_metadata.v3.definition import ( + CORE_AND_EXTENSIONS_CONTEXT, + ZARR_PYTHON_CONTEXT, + Chunk, + CodecDefinition, + DataTypeDefinition, + NumberClass, + ZarrV3Context, +) + +if TYPE_CHECKING: + from zarr_metadata.typed_json import JSONValue + +NUMBER_CLASSES = { + **dict.fromkeys( + [ + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + ], + "integral", + ), + **dict.fromkeys(["float16", "float32", "float64"], "floating_point"), + **dict.fromkeys(["complex64", "complex128"], "complex"), + **dict.fromkeys( + [ + "bool", + "bytes", + "fixed_length_utf32", + "null_terminated_bytes", + "numpy.datetime64", + "numpy.timedelta64", + "r*", + "raw_bytes", + "string", + "struct", + "variable_length_bytes", + ], + "non_numeric", + ), +} +"""The number class of each data type this package defines, by the name its definition is filed under.""" + + +def test_each_data_type_declares_its_number_class() -> None: + defined = {**CORE_AND_EXTENSIONS_CONTEXT.data_types, **ZARR_PYTHON_CONTEXT.data_types} + assert {name: definition.number_class for name, definition in defined.items()} == ( + NUMBER_CLASSES + ) + + +ACME_DATA_TYPES = [ + *( + DataTypeDefinition( + name=f"acme.{number_class}", configuration=EmptyConfiguration, number_class=number_class + ) + for number_class in get_args(NumberClass) + ), + DataTypeDefinition(name="acme.unknown", configuration=EmptyConfiguration), +] +"""An extension data type of each number class, and one that declares none.""" + +CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(data_types=ACME_DATA_TYPES)) + +LITTLE: JSONValue = {"name": "bytes", "configuration": {"endian": "little"}} + + +def _cast(data_type: JSONValue, **members: JSONValue) -> JSONValue: + return {"name": "cast_value", "configuration": {"data_type": data_type, **members}} + + +def _chunk(data_type: str) -> Chunk: + return Chunk((frozenset({4}),), resolve(data_type, DataTypeDefinition, CONTEXT)[0]) + + +_NO_REALS = 'expected a chunk of a data type that models real numbers, got "{}"' +_NO_REAL_TARGET = 'expected a data type that models real numbers, got "{}"' +_NO_SCALING = 'expected a chunk of an integer or floating-point data type, got "{}"' +_NO_WRAP = 'expected an integral data_type to wrap to, got "{}"' + + +@pytest.mark.parametrize( + ("codec", "data_type", "found"), + [ + # cast_value takes real numbers, and a data type nothing is known of. + (_cast("int8"), "acme.integral", []), + (_cast("int8"), "acme.floating_point", []), + (_cast("int8"), "acme.unknown", []), + (_cast("int8"), "acme.complex", [((0,), _NO_REALS.format("acme.complex"))]), + (_cast("int8"), "acme.non_numeric", [((0,), _NO_REALS.format("acme.non_numeric"))]), + # It casts to the same data types it casts from. + (_cast("acme.integral"), "int8", []), + (_cast("acme.floating_point"), "int8", []), + (_cast("acme.unknown"), "int8", []), + ( + _cast("acme.complex"), + "int8", + [((0, "configuration", "data_type"), _NO_REAL_TARGET.format("acme.complex"))], + ), + ( + _cast("acme.non_numeric"), + "int8", + [((0, "configuration", "data_type"), _NO_REAL_TARGET.format("acme.non_numeric"))], + ), + # It wraps to an integral data type, and to one nothing is known of. + (_cast("acme.integral", out_of_range="wrap"), "int8", []), + (_cast("acme.unknown", out_of_range="wrap"), "int8", []), + ( + _cast("acme.floating_point", out_of_range="wrap"), + "int8", + [((0, "configuration", "out_of_range"), _NO_WRAP.format("acme.floating_point"))], + ), + # scale_offset takes the same data types. + ("scale_offset", "acme.integral", []), + ("scale_offset", "acme.floating_point", []), + ("scale_offset", "acme.unknown", []), + ("scale_offset", "acme.complex", [((0,), _NO_SCALING.format("acme.complex"))]), + ("scale_offset", "acme.non_numeric", [((0,), _NO_SCALING.format("acme.non_numeric"))]), + ], +) +def test_the_arithmetic_codecs_read_the_number_class( + codec: JSONValue, data_type: str, found: list[tuple[tuple[str | int, ...], str]] +) -> None: + read = [ + resolve(each, CodecDefinition, CONTEXT, (index,)) + for index, each in enumerate([codec, LITTLE]) + ] + _, in_pipeline = read_pipeline([each for each, _ in read], _chunk(data_type)) + problems = [*(problem for _, own in read for problem in own), *in_pipeline] + assert [(problem.loc, problem.message) for problem in problems] == found + + +@pytest.mark.parametrize("number_class", ["real", 1, ("integral",)]) +def test_error_a_number_class_that_is_not_one(number_class: object) -> None: + with pytest.raises(TypeError, match="'acme.d': number_class is one of .* or None, got"): + DataTypeDefinition( + name="acme.d", + configuration=EmptyConfiguration, + number_class=number_class, # pyright: ignore[reportArgumentType] + ) diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 8e73731d38..7fcab1c1d2 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -28,6 +28,9 @@ ) from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, + ArrayArrayCodecDefinition, + ArrayBytesCodecDefinition, + BytesBytesCodecDefinition, Chunk, CodecDefinition, DataTypeDefinition, @@ -51,17 +54,16 @@ def _tiled(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> C return Chunk(doubled, chunk.data_type) -ACME_TILE = CodecDefinition( +ACME_TILE = ArrayArrayCodecDefinition( name="acme.tile", configuration=EmptyConfiguration, - kind="array_array", size="static", transition=_tiled, ) """An array -> array codec that declares its output chunk.""" -ACME_SHUFFLE = CodecDefinition( - name="acme.shuffle", configuration=EmptyConfiguration, kind="array_array", size="static" +ACME_SHUFFLE = ArrayArrayCodecDefinition( + name="acme.shuffle", configuration=EmptyConfiguration, size="static" ) """An array -> array codec that does not declare its output chunk.""" @@ -187,20 +189,6 @@ def _lengths(*axes: set[int]) -> tuple[frozenset[int], ...]: [CHUNK, Chunk(), Chunk(None, _of("float32").data_type)], ), ([_scale(offset=1), "acme.vlen"], Chunk(), [Chunk(), Chunk()]), - # Where the specs disagree or say nothing, a data type is not checked. - # The numpy time types say they accept any codec of 64-bit integers, - # and scale_offset does not say whether it accepts complex numbers. - ( - [_cast("int64"), LITTLE], - _of({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}), - [ - _of( - {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}} - ), - _of("int64"), - ], - ), - ([_scale(offset=[1, 0]), LITTLE], _of("complex64"), [_of("complex64"), _of("complex64")]), ], ) def test_every_pipeline_hands_each_codec_the_chunk_the_one_before_hands_on( @@ -323,8 +311,21 @@ def test_error_a_scalar_the_cast_maps_from_that_is_not_of_the_data_type_it_is_ha @pytest.mark.parametrize("codec", [_cast("float32"), _scale()]) -@pytest.mark.parametrize("data_type", ["bool", "string", _struct("int8")]) -def test_error_an_arithmetic_codec_handed_values_that_are_no_numbers( +@pytest.mark.parametrize( + "data_type", + [ + "bool", + "string", + _struct("int8"), + # Each codec's spec lists the integer and floating-point types among + # the data types of its repository that it supports, which leaves out + # the complex types and the numpy time types. + "complex64", + {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}, + {"name": "numpy.timedelta64", "configuration": {"unit": "s", "scale_factor": 1}}, + ], +) +def test_error_an_arithmetic_codec_handed_a_data_type_its_spec_does_not_list( codec: JSONValue, data_type: JSONValue ) -> None: # The codec comes before an array -> bytes codec that nothing in scope @@ -341,10 +342,6 @@ def test_error_a_codec_handed_on_what_another_could_not_take_is_judged_too() -> ] -def test_error_a_cast_handed_complex_numbers() -> None: - assert _problems([_cast("float32"), "acme.vlen"], _of("complex64")) == [((0,), "invalid_value")] - - @pytest.mark.parametrize( ("members", "data_type", "found"), [ @@ -360,10 +357,9 @@ def test_error_a_scale_or_offset_that_is_not_a_value_of_the_data_type_it_is_hand def test_error_a_transition_that_gives_something_else() -> None: - lying = CodecDefinition( + lying = ArrayArrayCodecDefinition( name="acme.lying", configuration=EmptyConfiguration, - kind="array_array", size="static", transition=lambda configuration, nested, chunk: "a chunk", # pyright: ignore[reportArgumentType] ) @@ -376,10 +372,9 @@ def test_error_a_transition_that_raises_says_whose_it_is() -> None: def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> Chunk: raise KeyError("axis") - raising = CodecDefinition( + raising = ArrayArrayCodecDefinition( name="acme.raising", configuration=EmptyConfiguration, - kind="array_array", size="static", transition=transition, ) @@ -409,10 +404,9 @@ def test_error_a_transition_that_builds_a_chunk_of_something_else( def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> Chunk: return Chunk(lengths, data_type) # pyright: ignore[reportArgumentType] - odd = CodecDefinition( + odd = ArrayArrayCodecDefinition( name="acme.odd", configuration=EmptyConfiguration, - kind="array_array", size="static", transition=transition, ) @@ -427,24 +421,20 @@ def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) @pytest.mark.parametrize( ("kind", "attribute"), [ - ("bytes_bytes", "chunk_rules"), - ("bytes_bytes", "transition"), - ("bytes_bytes", "pipelines"), - ("array_bytes", "transition"), + (BytesBytesCodecDefinition, "chunk_rules"), + (BytesBytesCodecDefinition, "transition"), + (BytesBytesCodecDefinition, "pipelines"), + (ArrayBytesCodecDefinition, "transition"), ], ) -def test_error_a_codec_hook_nothing_would_ask(kind: str, attribute: str) -> None: +def test_error_a_codec_hook_nothing_would_ask( + kind: type[CodecDefinition[Any]], attribute: str +) -> None: # A bytes -> bytes codec receives bytes, and only an array -> array - # codec outputs a chunk, so these hooks would never be called. + # codec outputs a chunk, so their kinds have no such hooks. hook = {attribute: lambda configuration, nested, chunk: ()} - with pytest.raises(TypeError, match=f"'acme.x': {attribute}, "): - CodecDefinition( - name="acme.x", - configuration=EmptyConfiguration, - kind=kind, # pyright: ignore[reportArgumentType] - size="static", - **hook, # pyright: ignore[reportArgumentType] - ) + with pytest.raises(TypeError, match=f"unexpected keyword argument '{attribute}'"): + kind(name="acme.x", configuration=EmptyConfiguration, size="static", **hook) # pyright: ignore[reportArgumentType] def test_an_array_document_s_codecs_are_each_judged_against_the_chunk_they_are_handed() -> None: @@ -489,10 +479,9 @@ def _holder( pipelines: object, types: JSONValue = ("uint8",) ) -> ResolvedDeclaration[CodecDefinition[Any]]: """A codec that holds a pipeline of codecs and a list of data types, `types`, and whose `pipelines` hook is `pipelines`.""" - holder = CodecDefinition( + holder = ArrayBytesCodecDefinition( name="acme.holder", configuration=AcmeHolderConfiguration, - kind="array_bytes", size="dynamic", pipelines=pipelines, # pyright: ignore[reportArgumentType] ) diff --git a/packages/zarr-metadata/tests/v3/test_scope.py b/packages/zarr-metadata/tests/v3/test_scope.py index 1feca41d63..5ccc6b425d 100644 --- a/packages/zarr-metadata/tests/v3/test_scope.py +++ b/packages/zarr-metadata/tests/v3/test_scope.py @@ -32,6 +32,7 @@ CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, ArrayExtraFieldDefinition, + BytesBytesCodecDefinition, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, @@ -50,9 +51,7 @@ }, } -MY_GZIP = CodecDefinition( - name="gzip", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" -) +MY_GZIP = BytesBytesCodecDefinition(name="gzip", configuration=EmptyConfiguration, size="dynamic") """A private definition of the name `gzip`: a second definition under the same name, which takes no configuration. Definitions compare by their contents, so a definition rebuilt from the @@ -109,9 +108,9 @@ def test_error_a_scope_conflict_says_each_disagreement() -> None: ) assert error.conflicts[0].key == (CodecDefinition, "bytes") assert str(error) == ( - "codec 'bytes' at ('codecs', 0): claimed CodecDefinition(name='bytes') of " + "codec 'bytes' at ('codecs', 0): claimed ArrayBytesCodecDefinition(name='bytes') of " "BytesCodecConfiguration, found no definition; " - "codec 'gzip': claimed no definition, found CodecDefinition(name='gzip') of " + "codec 'gzip': claimed no definition, found BytesBytesCodecDefinition(name='gzip') of " "GzipCodecConfiguration" ) @@ -448,9 +447,7 @@ def test_a_gain_is_judged_by_what_the_definition_reads_not_by_spelling() -> None assert refines(read, unread) -OTHER_GZIP = CodecDefinition( - name="gzip", configuration=EmptyConfiguration, kind="bytes_bytes", size="static" -) +OTHER_GZIP = BytesBytesCodecDefinition(name="gzip", configuration=EmptyConfiguration, size="static") """A second definition under the core gzip's name, which makes a union conflict.""" diff --git a/pyproject.toml b/pyproject.toml index 983ef58c96..839b5fdef8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ maintainers = [ ] requires-python = ">=3.12" dependencies = [ - 'packaging>=22.0', + 'packaging>=24.0', 'numpy>=2', # 0.16.6 and 0.17.1 refuse to unpickle untrusted data (GHSA-jqc8-qjfv-5m6r); 0.17.0 does not. 'numcodecs>=0.16.6,!=0.17.0', @@ -115,7 +115,6 @@ test = [ "pytest==9.2.0", "pytest-asyncio==1.4.1.post1", "pytest-cov==7.1.0", - "pytest-accept==0.3.0", "numpydoc==1.11.0", "hypothesis==6.170.14", "pytest-reportlog==1.0.0", @@ -251,7 +250,7 @@ python = "3.12" features = ["remote"] dependency-groups = ["remote-tests"] extra-dependencies = [ - 'packaging==22.*', + 'packaging==24.*', 'numpy==2.0.*', 'numcodecs==0.16.*', 'fsspec==2023.10.0', @@ -263,10 +262,6 @@ extra-dependencies = [ 'msgspec==0.19.*', ] -[tool.hatch.envs.min_deps.env-vars] -# Keep pytest<9.2 here so packaging==22.* (our declared floor) still resolves. -UV_OVERRIDE = "{root}/ci/min_deps-overrides.txt" - [tool.hatch.envs.default] installer = "uv" @@ -462,10 +457,6 @@ filterwarnings = [ # loop-lifecycle changes. "ignore:Exception ignored in[\\s\\S]*=9.2 applies these filters while importing plugins (pytest#12697). pytest-accept - # imports astor, which looks up the deprecated ast.Num at import time on Python < 3.14, - # so "error" above would abort the run before collection. - "ignore:ast\\.\\w+ is deprecated:DeprecationWarning:astor", ] markers = [ "gpu: mark a test as requiring CuPy and GPU", diff --git a/src/zarr/metadata/migrate_v3.py b/src/zarr/metadata/migrate_v3.py index ad177e19fb..cd94f12d84 100644 --- a/src/zarr/metadata/migrate_v3.py +++ b/src/zarr/metadata/migrate_v3.py @@ -209,12 +209,18 @@ def _convert_array_metadata(metadata_v2: ArrayV2Metadata) -> ArrayV3Metadata: bytes_bytes_codec = _convert_compressor(metadata_v2.compressor, metadata_v2.dtype) codecs.append(bytes_bytes_codec) + # Zarr v2 allows a null fill value, Zarr v3 does not. Use the default that is + # already used when reading chunks of the v2 array that were never written. + fill_value: object = metadata_v2.fill_value + if fill_value is None: + fill_value = metadata_v2.dtype.default_scalar() + return ArrayV3Metadata( shape=metadata_v2.shape, data_type=metadata_v2.dtype, chunk_grid=RegularChunkGridMetadata(chunk_shape=metadata_v2.chunks), chunk_key_encoding=chunk_key_encoding, - fill_value=metadata_v2.fill_value, + fill_value=fill_value, codecs=codecs, attributes=metadata_v2.attributes, dimension_names=None, diff --git a/tests/test_cli/test_migrate_v3.py b/tests/test_cli/test_migrate_v3.py index 83401487db..2707341e1d 100644 --- a/tests/test_cli/test_migrate_v3.py +++ b/tests/test_cli/test_migrate_v3.py @@ -455,6 +455,42 @@ def test_migrate_endian( assert np.all(zarr_array[:] == 1) +@pytest.mark.parametrize( + ("dtype", "expected_fill_value"), + [(" None: + """A null v2 fill value becomes the default that is used when reading the v2 array. + + The first chunk is only partly written, so its stored padding must read back + the same after migration. The second chunk is never written. + """ + zarr_array = zarr.create_array( + store=local_store, + shape=(10,), + chunks=(5,), + dtype=dtype, + zarr_format=2, + fill_value=None, + ) + zarr_array[:3] = 1 + assert zarr_array.metadata.fill_value is None + v2_values = zarr_array[:] + + result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) + assert result.exit_code == 0 + assert (local_store.root / "zarr.json").exists() + + zarr_array = zarr.open_array(local_store.root, zarr_format=3) + assert zarr_array.metadata.fill_value == expected_fill_value + expected = np.array([1] * 3 + [expected_fill_value] * 7, dtype=dtype) + assert np.array_equal(v2_values, expected) + assert np.array_equal(zarr_array[:], expected) + + @pytest.mark.parametrize("node_type", ["array", "group"]) def test_migrate_v3(local_store: LocalStore, node_type: str) -> None: """Attempting to convert a v3 array/group should always fail""" diff --git a/uv.lock b/uv.lock index 38e8220a06..8dbf06aab4 100644 --- a/uv.lock +++ b/uv.lock @@ -297,15 +297,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/25/4235f4b375a67a37ac595bf478a5407a744eaf7b64113c1c3a5432b8d954/ast_serialize-0.13.1-cp39-abi3-win_arm64.whl", hash = "sha256:48f11d96c9ec71119612f8e79a8c55c12f2b136716d68347433c561d27a25f43", size = 1139035, upload-time = "2026-10-17T00:04:15.767Z" }, ] -[[package]] -name = "astor" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, -] - [[package]] name = "astroid" version = "4.3.4" @@ -2837,19 +2828,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl", hash = "sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588", size = 411776, upload-time = "2026-10-27T21:30:19.131Z" }, ] -[[package]] -name = "pytest-accept" -version = "0.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astor" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/42/573e6dcd8d1a5fc779d525268d5036f710d018cdeefbae2bd5559687a6d5/pytest_accept-0.3.0.tar.gz", hash = "sha256:6f4e03e2492621e10b7678a331828274498cca04686b15b0f3e704e16e651068", size = 29744, upload-time = "2026-06-11T18:44:39.03Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/40/79f6f13e616f580b9c35110376818c76cacc54b70d2965bf1f82be336c66/pytest_accept-0.3.0-py3-none-any.whl", hash = "sha256:5e73f27020853b861a63dc630dfe63124af4bd60e0e0bb692e1376b99798f2b5", size = 39846, upload-time = "2026-06-11T18:44:37.965Z" }, -] - [[package]] name = "pytest-asyncio" version = "1.4.1.post1" @@ -3872,7 +3850,6 @@ dev = [ { name = "numpydoc" }, { name = "obstore" }, { name = "pytest" }, - { name = "pytest-accept" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, { name = "pytest-codspeed" }, @@ -3914,7 +3891,6 @@ remote-tests = [ { name = "numpydoc" }, { name = "obstore" }, { name = "pytest" }, - { name = "pytest-accept" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, { name = "pytest-codspeed" }, @@ -3930,7 +3906,6 @@ test = [ { name = "hypothesis" }, { name = "numpydoc" }, { name = "pytest" }, - { name = "pytest-accept" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, { name = "pytest-codspeed" }, @@ -3951,7 +3926,7 @@ requires-dist = [ { name = "numcodecs", specifier = ">=0.16.6,!=0.17.0" }, { name = "numpy", specifier = ">=2" }, { name = "obstore", marker = "extra == 'remote'", specifier = ">=0.5.1" }, - { name = "packaging", specifier = ">=22.0" }, + { name = "packaging", specifier = ">=24.0" }, { name = "typer", marker = "extra == 'cli'" }, { name = "typing-extensions", specifier = ">=4.14" }, { name = "universal-pathlib", marker = "extra == 'optional'" }, @@ -3979,7 +3954,6 @@ dev = [ { name = "numpydoc", specifier = "==1.11.0" }, { name = "obstore", specifier = ">=0.5.1" }, { name = "pytest", specifier = "==9.2.0" }, - { name = "pytest-accept", specifier = "==0.3.0" }, { name = "pytest-asyncio", specifier = "==1.4.1.post1" }, { name = "pytest-benchmark", specifier = "==5.3.1" }, { name = "pytest-codspeed", specifier = "==5.0.4" }, @@ -4019,7 +3993,6 @@ remote-tests = [ { name = "numpydoc", specifier = "==1.11.0" }, { name = "obstore", specifier = ">=0.5.1" }, { name = "pytest", specifier = "==9.2.0" }, - { name = "pytest-accept", specifier = "==0.3.0" }, { name = "pytest-asyncio", specifier = "==1.4.1.post1" }, { name = "pytest-benchmark", specifier = "==5.3.1" }, { name = "pytest-codspeed", specifier = "==5.0.4" }, @@ -4035,7 +4008,6 @@ test = [ { name = "hypothesis", specifier = "==6.170.14" }, { name = "numpydoc", specifier = "==1.11.0" }, { name = "pytest", specifier = "==9.2.0" }, - { name = "pytest-accept", specifier = "==0.3.0" }, { name = "pytest-asyncio", specifier = "==1.4.1.post1" }, { name = "pytest-benchmark", specifier = "==5.3.1" }, { name = "pytest-codspeed", specifier = "==5.0.4" },