From 53d57d210fcab85e8d1858bab6e5e0456a51e932 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 12 Nov 2026 14:22:08 +0100 Subject: [PATCH 01/11] refactor(zarr-metadata): move extension types into types-only modules Every extension module held its types (name constants, `Literal` names, configuration and object `TypedDict`s, metadata aliases) beside its definition. Codec modules imported data type modules at run time, so a data type that needed a codec's types could import them only under `TYPE_CHECKING`. The types now live in modules that hold types only and import no definition: one module per kind under `v3/_types/` (codec, data_type, chunk_grid, chunk_key_encoding, consolidated) and `v2/_types/` (codec, data_type). Their packages' `__init__`s hold no definitions, so importing a types module never runs one. `EmptyConfiguration` moves to `_common`, and `ZarrV2DataTypeDeclaration` to `v2/_declaration_json`. Rules and other functions stay beside their definition, which imports its types from the types module. Each public extension module re-exports its types, so every public import path is unchanged. The codec rules that name data types (`_arithmetic`, `vlen_utf8`, `vlen_bytes`, `sharding_indexed`) import those names from the types module. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/changes/XXXX.misc.md | 7 + .../src/zarr_metadata/_common.py | 13 +- .../src/zarr_metadata/definition/__init__.py | 2 +- .../zarr_metadata/definition/_definition.py | 9 +- .../src/zarr_metadata/testing/_codec.py | 3 +- .../src/zarr_metadata/testing/_data_type.py | 3 +- .../src/zarr_metadata/testing/_strategy.py | 2 +- .../src/zarr_metadata/v2/_declaration_json.py | 12 +- .../src/zarr_metadata/v2/_definition.py | 5 +- .../src/zarr_metadata/v2/_types/__init__.py | 6 + .../src/zarr_metadata/v2/_types/codec.py | 225 +++++ .../src/zarr_metadata/v2/_types/data_type.py | 122 +++ .../src/zarr_metadata/v2/codec/checksum.py | 17 +- .../src/zarr_metadata/v2/codec/compression.py | 117 +-- .../src/zarr_metadata/v2/codec/filters.py | 80 +- .../src/zarr_metadata/v2/codec/vlen.py | 12 +- .../src/zarr_metadata/v2/data_type/_rules.py | 8 +- .../zarr_metadata/v2/data_type/fixed_width.py | 5 +- .../src/zarr_metadata/v2/data_type/object.py | 13 +- .../src/zarr_metadata/v2/data_type/scalar.py | 43 +- .../src/zarr_metadata/v2/data_type/struct.py | 33 +- .../src/zarr_metadata/v2/data_type/time.py | 35 +- .../src/zarr_metadata/v2/definition.py | 7 +- .../src/zarr_metadata/v3/_json_schema.py | 8 +- .../src/zarr_metadata/v3/_types/__init__.py | 7 + .../src/zarr_metadata/v3/_types/chunk_grid.py | 141 +++ .../v3/_types/chunk_key_encoding.py | 147 ++++ .../src/zarr_metadata/v3/_types/codec.py | 809 ++++++++++++++++++ .../zarr_metadata/v3/_types/consolidated.py | 83 ++ .../src/zarr_metadata/v3/_types/data_type.py | 768 +++++++++++++++++ .../v3/chunk_grid/rectilinear.py | 69 +- .../zarr_metadata/v3/chunk_grid/regular.py | 63 +- .../v3/chunk_key_encoding/default.py | 68 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 68 +- .../src/zarr_metadata/v3/codec/_arithmetic.py | 22 +- .../src/zarr_metadata/v3/codec/blosc.py | 100 +-- .../src/zarr_metadata/v3/codec/bytes.py | 78 +- .../src/zarr_metadata/v3/codec/cast_value.py | 139 +-- .../src/zarr_metadata/v3/codec/crc32c.py | 53 +- .../src/zarr_metadata/v3/codec/gzip.py | 66 +- .../zarr_metadata/v3/codec/scale_offset.py | 79 +- .../v3/codec/sharding_indexed.py | 106 +-- .../src/zarr_metadata/v3/codec/transpose.py | 59 +- .../src/zarr_metadata/v3/codec/vlen_bytes.py | 49 +- .../src/zarr_metadata/v3/codec/vlen_utf8.py | 45 +- .../src/zarr_metadata/v3/codec/zstd.py | 75 +- .../src/zarr_metadata/v3/consolidated.py | 75 +- .../src/zarr_metadata/v3/data_type/_byte.py | 14 +- .../src/zarr_metadata/v3/data_type/_float.py | 19 +- .../zarr_metadata/v3/data_type/_numpy_time.py | 18 +- .../src/zarr_metadata/v3/data_type/bool.py | 15 +- .../src/zarr_metadata/v3/data_type/bytes.py | 30 +- .../zarr_metadata/v3/data_type/complex128.py | 33 +- .../zarr_metadata/v3/data_type/complex64.py | 33 +- .../v3/data_type/fixed_length_utf32.py | 44 +- .../src/zarr_metadata/v3/data_type/float16.py | 58 +- .../src/zarr_metadata/v3/data_type/float32.py | 58 +- .../src/zarr_metadata/v3/data_type/float64.py | 59 +- .../src/zarr_metadata/v3/data_type/int16.py | 21 +- .../src/zarr_metadata/v3/data_type/int32.py | 21 +- .../src/zarr_metadata/v3/data_type/int64.py | 21 +- .../src/zarr_metadata/v3/data_type/int8.py | 17 +- .../v3/data_type/null_terminated_bytes.py | 44 +- .../v3/data_type/numpy_datetime64.py | 60 +- .../v3/data_type/numpy_timedelta64.py | 60 +- .../src/zarr_metadata/v3/data_type/raw.py | 39 +- .../src/zarr_metadata/v3/data_type/string.py | 19 +- .../src/zarr_metadata/v3/data_type/struct.py | 68 +- .../src/zarr_metadata/v3/data_type/uint16.py | 21 +- .../src/zarr_metadata/v3/data_type/uint32.py | 21 +- .../src/zarr_metadata/v3/data_type/uint64.py | 21 +- .../src/zarr_metadata/v3/data_type/uint8.py | 21 +- .../v3/data_type/variable_length_bytes.py | 20 +- .../v3/data_type/zarr_python_raw_bytes.py | 44 +- .../tests/testing/test_strategies.py | 2 +- 75 files changed, 2772 insertions(+), 2085 deletions(-) create mode 100644 packages/zarr-metadata/changes/XXXX.misc.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/v2/_types/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v2/_types/codec.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v2/_types/data_type.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_types/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_key_encoding.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_types/consolidated.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py diff --git a/packages/zarr-metadata/changes/XXXX.misc.md b/packages/zarr-metadata/changes/XXXX.misc.md new file mode 100644 index 0000000000..46508e8d26 --- /dev/null +++ b/packages/zarr-metadata/changes/XXXX.misc.md @@ -0,0 +1,7 @@ +The types of every extension (each codec's, data type's, chunk grid's and +chunk key encoding's name constants, `Literal` types, configuration and object +`TypedDict`s and metadata aliases, in both formats) now live in private modules +that hold types only and import no definition. A definition of one kind can +therefore import the types of another kind at run time, as a data type that +names its default codec does. Every name keeps its public import paths, since +each extension's module re-exports its types. diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index a431c53880..ddf590e660 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -4,9 +4,12 @@ `JSONValue` is the type of any JSON value. `ZarrV3NamedConfigJSON` is the `{name, configuration, must_understand}` envelope of a v3 declaration. It is v3's, but it lives here because the shared declaration reader in -`zarr_metadata.definition` uses the envelope as its default. Public homes: -`zarr_metadata.typed_json` for `JSONValue`, `zarr_metadata.v3` for the -envelope. +`zarr_metadata.definition` uses the envelope as its default. +`EmptyConfiguration` is the configuration of an extension of either +format that has nothing to configure, so the types of both formats' +extensions import it. Public homes: `zarr_metadata.typed_json` for +`JSONValue`, `zarr_metadata.v3` for the envelope, and +`zarr_metadata.definition` for `EmptyConfiguration`. """ from collections.abc import Mapping @@ -83,3 +86,7 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): grid or chunk key encoding (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581). """ + + +class EmptyConfiguration(TypedDict, closed=True): + """The configuration of a definition with nothing to configure. Its declaration is written with its name only.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 2ba4055ba6..5e57c4a582 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -264,11 +264,11 @@ def acme_lz4_rules( built from it does. """ +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.definition._definition import ( AcceptedDeclaration, Context, Definition, - EmptyConfiguration, Nested, RefusedDeclaration, ResolvedDeclaration, diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 9e3d746d24..55496acbb3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -58,9 +58,9 @@ get_type_hints, ) -from typing_extensions import TypeAliasType, TypedDict, TypeVar, is_typeddict +from typing_extensions import TypeAliasType, TypeVar, is_typeddict -from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON +from zarr_metadata._common import EmptyConfiguration, JSONValue, ZarrV3NamedConfigJSON from zarr_metadata._json import ( Problems, ValidationProblem, @@ -125,10 +125,6 @@ def fill_value_as_written(configuration: object, nested: object, value: T) -> T: return value -class EmptyConfiguration(TypedDict, closed=True): - """The configuration of a definition with nothing to configure. Its declaration is written with its name only.""" - - _DECLARATION_KINDS: Final[dict[TypeAliasType, type[Definition[Any]]]] = {} """Maps each declaration alias to the kind that a member annotated with it is read as. Each kind adds its aliases when its class is built.""" @@ -1787,7 +1783,6 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "C", "Context", "Definition", - "EmptyConfiguration", "Nested", "Problems", "RefusedDeclaration", diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index 5d2d34f90e..2b26c320dd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -19,7 +19,8 @@ from hypothesis import reject from hypothesis import strategies as st -from zarr_metadata.definition._definition import AcceptedDeclaration, EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration +from zarr_metadata.definition._definition import AcceptedDeclaration from zarr_metadata.testing._data_type import ( COMPLEX_WIDTHS, FLOAT_WIDTHS, diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py index 89b1991777..1e577a9a64 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -27,8 +27,7 @@ ) from zarr_metadata.testing._strategy import CHARACTERS, EMPTY_CONFIGURATIONS, DataTypeStrategy from zarr_metadata.v3._definition import DataTypeDefinition, storage_of -from zarr_metadata.v3.data_type._float import FloatSpecialFillValue -from zarr_metadata.v3.data_type._numpy_time import NUMPY_TIME_UNIT +from zarr_metadata.v3._types.data_type import NUMPY_TIME_UNIT, FloatSpecialFillValue from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py index 4fdca11c51..e7bf514df8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py @@ -19,7 +19,7 @@ from hypothesis import strategies as st -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( Chunk, ChunkGridDefinition, diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_declaration_json.py b/packages/zarr-metadata/src/zarr_metadata/v2/_declaration_json.py index e6f9763b22..0996c50068 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_declaration_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_declaration_json.py @@ -1,4 +1,4 @@ -"""The JSON shapes of a v2 dtype and a v2 codec. They are in their own module so that both the array document and the definitions can import them without a cycle; `zarr_metadata.v2.data_type` and `zarr_metadata.v2.codec` export them.""" +"""The JSON shapes of a v2 dtype and a v2 codec, and the alias that annotates a member holding a v2 dtype. They are in their own module so that both the array document and the definitions can import them without a cycle; `zarr_metadata.v2.data_type` and `zarr_metadata.v2.codec` export the shapes, and `zarr_metadata.v2.definition` the alias.""" from typing_extensions import TypeAliasType, TypedDict @@ -43,4 +43,12 @@ class ZarrV2CodecMetadata(TypedDict, extra_items=JSONValue): """The numcodecs id that names the codec, such as `"blosc"`.""" -__all__ = ["ZarrV2CodecMetadata", "ZarrV2DataTypeMetadata"] +ZarrV2DataTypeDeclaration = TypeAliasType("ZarrV2DataTypeDeclaration", ZarrV2DataTypeMetadata) +"""A declaration of a v2 dtype: the value of a document's `dtype` member, or the type of a struct record. It is read in the same scope as its container.""" + + +__all__ = [ + "ZarrV2CodecMetadata", + "ZarrV2DataTypeDeclaration", + "ZarrV2DataTypeMetadata", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py index b3bd11dc12..ebe6fc1b04 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py @@ -31,7 +31,7 @@ Definition, WithFillValue, ) -from zarr_metadata.v2._declaration_json import ZarrV2DataTypeMetadata +from zarr_metadata.v2._declaration_json import ZarrV2DataTypeDeclaration if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -40,8 +40,6 @@ from zarr_metadata._json import Problems from zarr_metadata._typed_json import Loc -ZarrV2DataTypeDeclaration = TypeAliasType("ZarrV2DataTypeDeclaration", ZarrV2DataTypeMetadata) -"""A declaration of a v2 dtype: the value of a document's `dtype` member, or the type of a struct record. It is read in the same scope as its container.""" STRUCT_NAME: Final = "struct" """The name that a records array reads as. No typestr has this spelling.""" @@ -328,7 +326,6 @@ def _name_loc(cls, loc: Loc) -> Loc: "STRUCT_NAME", "TYPESTR_PATTERN", "ZarrV2CodecDefinition", - "ZarrV2DataTypeDeclaration", "ZarrV2DataTypeDefinition", "item_bytes", "parse_typestr", diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_types/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/_types/__init__.py new file mode 100644 index 0000000000..46151363c8 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_types/__init__.py @@ -0,0 +1,6 @@ +"""The types of the v2 codecs and data types, one module per kind. + +Every module in this package holds types only and imports no definition, +so a definition of any kind can import the types of any other at run +time. Each name is public in its own submodule, which re-exports it. +""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_types/codec.py b/packages/zarr-metadata/src/zarr_metadata/v2/_types/codec.py new file mode 100644 index 0000000000..33c45e175b --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_types/codec.py @@ -0,0 +1,225 @@ +"""The types of the v2 codecs: the parameters of each codec that numcodecs configures. + +This module holds types only and imports no definition. Each codec's +submodule of `zarr_metadata.v2.codec` re-exports its names, and holds the +codec's definition and rules. +""" + +from typing import Annotated, Literal, NotRequired + +from annotated_types import Ge, Interval, Le +from typing_extensions import ReadOnly, TypedDict + +from zarr_metadata._common import JSONValue + +# checksum + + +class ZarrV2Checksum32Parameters(TypedDict, closed=True): + """`numcodecs.CRC32(location=None)` and the other 32-bit checksums. `location` is where the checksum is placed. When it is omitted or null, numcodecs places it at the start for crc32 and adler32, and at the end for crc32c.""" + + location: NotRequired[ReadOnly[Literal["start", "end"] | None]] + """Where the checksum is placed in the encoded chunk, `"start"` or `"end"`. + + Absent or null means the start for `crc32` and `adler32`, and the end for `crc32c`. + """ + + +# compression + +ZarrV2CompressionLevel = Annotated[int, Interval(ge=-1, le=9)] +"""A zlib-style compression level: 0 to 9, or -1 for zlib's default. numcodecs writes the level as given.""" + +_CInt = Annotated[int, Interval(ge=-(2**31), le=2**31 - 1)] +"""An integer that numcodecs passes to C as an `int`: one that fits in 32 bits, with a sign. numcodecs raises `OverflowError` for any other.""" + + +class ZarrV2ZlibParameters(TypedDict, closed=True): + """`numcodecs.Zlib(level=1)`.""" + + level: NotRequired[ReadOnly[ZarrV2CompressionLevel]] + """The zlib compression level. Absent means 1.""" + + +class ZarrV2GzipParameters(TypedDict, closed=True): + """`numcodecs.GZip(level=1)`.""" + + level: NotRequired[ReadOnly[ZarrV2CompressionLevel]] + """The zlib compression level. Absent means 1.""" + + +class ZarrV2Bz2Parameters(TypedDict, closed=True): + """`numcodecs.BZ2(level=1)`: 1 to 9.""" + + level: NotRequired[ReadOnly[Annotated[int, Interval(ge=1, le=9)]]] + """The bzip2 compression level. Absent means 1.""" + + +class ZarrV2LzmaParameters(TypedDict, closed=True): + """`numcodecs.LZMA(format=1, check=-1, preset=None, filters=None)`, with the parameters as the `lzma` module accepts them.""" + + format: NotRequired[ReadOnly[Annotated[int, Interval(ge=0, le=3)]]] + """The container format, an `lzma.FORMAT_*` code. Absent means 1, `FORMAT_XZ`. + + 2 is `FORMAT_ALONE`, and 3 is `FORMAT_RAW`, which requires `filters`. 0, + `FORMAT_AUTO`, only decompresses: `lzma` refuses it when compressing. + """ + check: NotRequired[ReadOnly[int]] + """The integrity check, an `lzma.CHECK_*` code, or -1 for the format's default. Absent means -1. + + Only `FORMAT_XZ` takes a check other than -1 or `CHECK_NONE` (0). + """ + preset: NotRequired[ReadOnly[Annotated[int, Ge(0)] | None]] + """The compression preset, 0 to 9, optionally ORed with `lzma.PRESET_EXTREME`, or null for lzma's default, 6. + + Absent means null. `lzma` refuses a preset together with `filters`. + """ + filters: NotRequired[ReadOnly[tuple[JSONValue, ...] | None]] + """A custom filter chain, as `lzma` filter specifiers, or null to use `preset`. Absent means null.""" + + +ZarrV2BloscCName = Literal["blosclz", "lz4", "lz4hc", "snappy", "zlib", "zstd"] +"""The compressors inside Blosc.""" + + +class ZarrV2BloscParameters(TypedDict, closed=True): + """`numcodecs.Blosc(cname='lz4', clevel=5, shuffle=1, blocksize=0, typesize=None)`. + + For `shuffle`, -1 is automatic, 0 is none, 1 is byte shuffle, and 2 is + bit shuffle. numcodecs writes a `shuffle` given as `True` or `False` + as it was given, and reads it as 1 or 0. A `blocksize` of 0 lets Blosc + choose the block size. numcodecs reads any `blocksize` that fits in a + C `int`, negative ones included. Blosc reads any `blocksize` other + than 0 as a size in bytes, and raises one below its minimum of 128 + bytes to that minimum, so a negative `blocksize` does not mean + automatic. + """ + + cname: NotRequired[ReadOnly[ZarrV2BloscCName]] + """The compressor that Blosc runs. Absent means `"lz4"`.""" + clevel: NotRequired[ReadOnly[Annotated[int, Interval(ge=0, le=9)]]] + """The compression level. Absent means 5. At 0, Blosc stores the data uncompressed.""" + shuffle: NotRequired[ReadOnly[Literal[-1, 0, 1, 2] | bool]] + """How Blosc shuffles the data before compressing it. Absent means 1, byte shuffle.""" + blocksize: NotRequired[ReadOnly[_CInt]] + """The size in bytes of the blocks that Blosc compresses each chunk in. Absent means 0, which lets Blosc choose.""" + typesize: NotRequired[ReadOnly[Annotated[int, Interval(ge=1, le=2**31 - 1)] | None]] + """The size in bytes of one element, the unit that Blosc shuffles by. + + Absent or null means the item size of the buffer that numcodecs is handed. + """ + + +class ZarrV2ZstdParameters(TypedDict, closed=True): + """`numcodecs.Zstd(level=0, checksum=False)`. Level 0 is zstd's default level, and negative levels give up compression ratio for speed. zstd clamps a level outside its own range, -131072 to 22, to that range, so numcodecs reads any level that fits in a C `int`.""" + + level: NotRequired[ReadOnly[_CInt]] + """The zstd compression level. Absent means 0, zstd's default level.""" + checksum: NotRequired[ReadOnly[bool]] + """Whether zstd writes a checksum of the content into each frame. Absent means false.""" + + +class ZarrV2Lz4Parameters(TypedDict, closed=True): + """`numcodecs.LZ4(acceleration=1)`. numcodecs passes `acceleration` to C as an `int`, and lz4 reads one below 1 as 1.""" + + acceleration: NotRequired[ReadOnly[_CInt]] + """How far lz4 trades compression ratio for speed: a larger value is faster. Absent means 1.""" + + +# filters + + +class ZarrV2ShuffleParameters(TypedDict, closed=True): + """`numcodecs.Shuffle(elementsize=4)`. An element size of 1 or less leaves the bytes as they are. numcodecs passes a larger one to C as a `Py_ssize_t`, so it is at most 2**63 - 1 on a 64-bit platform, and raises `OverflowError` for any other.""" + + elementsize: NotRequired[ReadOnly[Annotated[int, Le(2**63 - 1)]]] + """The size in bytes of the elements whose bytes are shuffled. Absent means 4.""" + + +class ZarrV2DeltaParameters(TypedDict, closed=True): + """`numcodecs.Delta(dtype, astype=None)`. When `astype` is omitted or null, it equals `dtype`.""" + + dtype: ReadOnly[str] + """The dtype of the decoded data.""" + astype: NotRequired[ReadOnly[str | None]] + """The dtype of the encoded data. Absent or null means `dtype`.""" + + +class ZarrV2FixedScaleOffsetParameters(TypedDict, closed=True): + """`numcodecs.FixedScaleOffset(offset, scale, dtype, astype=None)`. When `astype` is omitted or null, it equals `dtype`.""" + + offset: ReadOnly[float | int] + """The value subtracted from each element before it is scaled.""" + scale: ReadOnly[float | int] + """The factor that each element is multiplied by after the offset is subtracted. The product is rounded to the nearest integer.""" + dtype: ReadOnly[str] + """The dtype of the decoded data.""" + astype: NotRequired[ReadOnly[str | None]] + """The dtype of the encoded data. Absent or null means `dtype`.""" + + +class ZarrV2QuantizeParameters(TypedDict, closed=True): + """`numcodecs.Quantize(digits, dtype, astype=None)`: `dtype`, and `astype` when present, name a float. When `astype` is omitted or null, it equals `dtype`. + + `digits` is the number of decimal digits to keep, and may be negative. + numcodecs computes `10.0 ** -digits` and a power of two above it, which + overflow a float outside -308 to 307. + """ + + digits: ReadOnly[Annotated[int, Interval(ge=-308, le=307)]] + """The number of decimal digits of precision to keep.""" + dtype: ReadOnly[str] + """The float dtype of the decoded data.""" + astype: NotRequired[ReadOnly[str | None]] + """The float dtype of the encoded data. Absent or null means `dtype`.""" + + +class ZarrV2BitRoundParameters(TypedDict, closed=True): + """`numcodecs.BitRound(keepbits)`.""" + + keepbits: ReadOnly[Annotated[int, Ge(0)]] + """The number of mantissa bits to keep. + + numcodecs refuses, when encoding, a value above the mantissa width of the + data's float type: 10, 23 or 52 bits. + """ + + +class ZarrV2AsTypeParameters(TypedDict, closed=True): + """`numcodecs.AsType(encode_dtype, decode_dtype)`.""" + + encode_dtype: ReadOnly[str] + """The dtype of the encoded data.""" + decode_dtype: ReadOnly[str] + """The dtype of the decoded data.""" + + +# vlen + + +class ZarrV2VLenArrayParameters(TypedDict, closed=True): + """`numcodecs.VLenArray(dtype)`: the element type of each array.""" + + dtype: ReadOnly[str] + """The dtype of the elements of each variable-length array.""" + + +__all__ = [ + "ZarrV2AsTypeParameters", + "ZarrV2BitRoundParameters", + "ZarrV2BloscCName", + "ZarrV2BloscParameters", + "ZarrV2Bz2Parameters", + "ZarrV2Checksum32Parameters", + "ZarrV2CompressionLevel", + "ZarrV2DeltaParameters", + "ZarrV2FixedScaleOffsetParameters", + "ZarrV2GzipParameters", + "ZarrV2Lz4Parameters", + "ZarrV2LzmaParameters", + "ZarrV2QuantizeParameters", + "ZarrV2ShuffleParameters", + "ZarrV2VLenArrayParameters", + "ZarrV2ZlibParameters", + "ZarrV2ZstdParameters", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_types/data_type.py b/packages/zarr-metadata/src/zarr_metadata/v2/_types/data_type.py new file mode 100644 index 0000000000..7e779ca4ec --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_types/data_type.py @@ -0,0 +1,122 @@ +"""The types of the v2 data types: the configuration each family of typestrs is read into, and the JSON shapes of their fill values. + +This module holds types only and imports no definition. Each data type's +submodule of `zarr_metadata.v2.data_type` re-exports its names, and holds +the data type's definition and rules. +""" + +from typing import Annotated, Literal, NotRequired + +from annotated_types import Ge +from typing_extensions import ReadOnly, TypedDict + +from zarr_metadata._numpy import NumpyTimeScaleFactor, NumpyTimeTicks +from zarr_metadata.v2._declaration_json import ZarrV2DataTypeDeclaration + +# scalar + +ZarrV2ByteOrder = Literal["<", ">", "|"] +"""The byte orders that a typestr writes: little-endian, big-endian, and not relevant.""" + + +class ZarrV2ScalarConfiguration(TypedDict, closed=True): + """The configuration that a scalar typestr carries: its byte order and its size in bytes, such as `{"byteorder": "<", "itemsize": 4}` for ``.""" + itemsize: ReadOnly[Annotated[int, Ge(0)]] + """The size in bytes that the typestr writes, which must be 8.""" + unit: NotRequired[ReadOnly[str]] + """The NumPy time unit in brackets, such as `ns`. The canonical form writes `μs` as `us`. + + An array's dtype must have one. Only a codec's dtype parameter may leave + it out, as in ` ZarrV2ZlibParameters: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py index f840ef531f..f69b7cbe41 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py @@ -9,81 +9,21 @@ from __future__ import annotations -from typing import Annotated, Final, NotRequired +from typing import Final -from annotated_types import Ge, Interval, Le -from typing_extensions import ReadOnly, TypedDict - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v2._definition import ZarrV2CodecDefinition +from zarr_metadata.v2._types.codec import ( + ZarrV2AsTypeParameters, + ZarrV2BitRoundParameters, + ZarrV2DeltaParameters, + ZarrV2FixedScaleOffsetParameters, + ZarrV2QuantizeParameters, + ZarrV2ShuffleParameters, +) from zarr_metadata.v2.codec._dtype import canonical_typestr, dtype_parameter -class ZarrV2ShuffleParameters(TypedDict, closed=True): - """`numcodecs.Shuffle(elementsize=4)`. An element size of 1 or less leaves the bytes as they are. numcodecs passes a larger one to C as a `Py_ssize_t`, so it is at most 2**63 - 1 on a 64-bit platform, and raises `OverflowError` for any other.""" - - elementsize: NotRequired[ReadOnly[Annotated[int, Le(2**63 - 1)]]] - """The size in bytes of the elements whose bytes are shuffled. Absent means 4.""" - - -class ZarrV2DeltaParameters(TypedDict, closed=True): - """`numcodecs.Delta(dtype, astype=None)`. When `astype` is omitted or null, it equals `dtype`.""" - - dtype: ReadOnly[str] - """The dtype of the decoded data.""" - astype: NotRequired[ReadOnly[str | None]] - """The dtype of the encoded data. Absent or null means `dtype`.""" - - -class ZarrV2FixedScaleOffsetParameters(TypedDict, closed=True): - """`numcodecs.FixedScaleOffset(offset, scale, dtype, astype=None)`. When `astype` is omitted or null, it equals `dtype`.""" - - offset: ReadOnly[float | int] - """The value subtracted from each element before it is scaled.""" - scale: ReadOnly[float | int] - """The factor that each element is multiplied by after the offset is subtracted. The product is rounded to the nearest integer.""" - dtype: ReadOnly[str] - """The dtype of the decoded data.""" - astype: NotRequired[ReadOnly[str | None]] - """The dtype of the encoded data. Absent or null means `dtype`.""" - - -class ZarrV2QuantizeParameters(TypedDict, closed=True): - """`numcodecs.Quantize(digits, dtype, astype=None)`: `dtype`, and `astype` when present, name a float. When `astype` is omitted or null, it equals `dtype`. - - `digits` is the number of decimal digits to keep, and may be negative. - numcodecs computes `10.0 ** -digits` and a power of two above it, which - overflow a float outside -308 to 307. - """ - - digits: ReadOnly[Annotated[int, Interval(ge=-308, le=307)]] - """The number of decimal digits of precision to keep.""" - dtype: ReadOnly[str] - """The float dtype of the decoded data.""" - astype: NotRequired[ReadOnly[str | None]] - """The float dtype of the encoded data. Absent or null means `dtype`.""" - - -class ZarrV2BitRoundParameters(TypedDict, closed=True): - """`numcodecs.BitRound(keepbits)`.""" - - keepbits: ReadOnly[Annotated[int, Ge(0)]] - """The number of mantissa bits to keep. - - numcodecs refuses, when encoding, a value above the mantissa width of the - data's float type: 10, 23 or 52 bits. - """ - - -class ZarrV2AsTypeParameters(TypedDict, closed=True): - """`numcodecs.AsType(encode_dtype, decode_dtype)`.""" - - encode_dtype: ReadOnly[str] - """The dtype of the encoded data.""" - decode_dtype: ReadOnly[str] - """The dtype of the decoded data.""" - - def _shuffle_canonical(configuration: ZarrV2ShuffleParameters) -> ZarrV2ShuffleParameters: """Fill in the default `elementsize`, 4, as numcodecs writes it.""" return {"elementsize": 4, **configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py index 4497ab64e5..8b40b55ca5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py @@ -4,20 +4,12 @@ from typing import Final -from typing_extensions import ReadOnly, TypedDict - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v2._definition import ZarrV2CodecDefinition +from zarr_metadata.v2._types.codec import ZarrV2VLenArrayParameters from zarr_metadata.v2.codec._dtype import canonical_typestr, dtype_parameter -class ZarrV2VLenArrayParameters(TypedDict, closed=True): - """`numcodecs.VLenArray(dtype)`: the element type of each array.""" - - dtype: ReadOnly[str] - """The dtype of the elements of each variable-length array.""" - - def _vlen_array_canonical(configuration: ZarrV2VLenArrayParameters) -> ZarrV2VLenArrayParameters: """Write the `dtype` as NumPy spells it, as numcodecs writes it.""" return {**configuration, "dtype": canonical_typestr(configuration["dtype"])} diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/_rules.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/_rules.py index 6d6114879b..dde5fbe924 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/_rules.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/_rules.py @@ -13,9 +13,11 @@ from collections.abc import Iterator from zarr_metadata.definition._definition import Nested - from zarr_metadata.v2.data_type.fixed_width import ZarrV2Base64FillValue - from zarr_metadata.v2.data_type.scalar import ZarrV2ScalarConfiguration - from zarr_metadata.v2.data_type.time import ZarrV2TimeConfiguration + from zarr_metadata.v2._types.data_type import ( + ZarrV2Base64FillValue, + ZarrV2ScalarConfiguration, + ZarrV2TimeConfiguration, + ) @dataclass(frozen=True, slots=True) diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/fixed_width.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/fixed_width.py index 264aba2df4..618444a0ad 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/fixed_width.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/fixed_width.py @@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, Final from zarr_metadata.v2._definition import CODE_POINT_BYTES, ZarrV2DataTypeDefinition +from zarr_metadata.v2._types.data_type import ZarrV2Base64FillValue, ZarrV2ScalarConfiguration from zarr_metadata.v2.data_type._rules import orderless_at, sized, sized_base64_rules -from zarr_metadata.v2.data_type.scalar import ZarrV2ScalarConfiguration if TYPE_CHECKING: from collections.abc import Iterator @@ -14,9 +14,6 @@ from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested -ZarrV2Base64FillValue = str | None -"""A fill value that the v2 spec encodes as base64, for a fixed-length byte string, void, or a structured type (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L191-L193), or null.""" - def _void_fill_value_rules( configuration: ZarrV2ScalarConfiguration, nested: Nested, value: ZarrV2Base64FillValue diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py index 9e4dbf4605..15d25e688b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py @@ -4,19 +4,8 @@ from typing import Final -from typing_extensions import ReadOnly, TypedDict - from zarr_metadata.v2._definition import ZarrV2DataTypeDefinition -from zarr_metadata.v2.data_type.scalar import ( - ZarrV2ByteOrder, -) - - -class ZarrV2ObjectConfiguration(TypedDict, closed=True): - """The configuration that `|O` carries: a byte order, which the type ignores.""" - - byteorder: ReadOnly[ZarrV2ByteOrder] - """The byte order that the typestr writes. The canonical form writes `|`, as NumPy does.""" +from zarr_metadata.v2._types.data_type import ZarrV2ObjectConfiguration def _canonical(configuration: ZarrV2ObjectConfiguration) -> ZarrV2ObjectConfiguration: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py index 2f9ec2393b..35de7dea97 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py @@ -5,13 +5,18 @@ import math import struct from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated, Final, Literal - -from annotated_types import Ge -from typing_extensions import ReadOnly, TypedDict +from typing import TYPE_CHECKING, Final from zarr_metadata._json import ValidationProblem from zarr_metadata.v2._definition import ZarrV2DataTypeDefinition +from zarr_metadata.v2._types.data_type import ( + ZarrV2ByteOrder, + ZarrV2ComplexComponent, + ZarrV2ComplexFillValue, + ZarrV2FloatFillValue, + ZarrV2FloatSpecial, + ZarrV2ScalarConfiguration, +) from zarr_metadata.v2.data_type._rules import orderless_at, sized if TYPE_CHECKING: @@ -19,23 +24,6 @@ from zarr_metadata.definition._definition import Nested -ZarrV2ByteOrder = Literal["<", ">", "|"] -"""The byte orders that a typestr writes: little-endian, big-endian, and not relevant.""" - - -class ZarrV2ScalarConfiguration(TypedDict, closed=True): - """The configuration that a scalar typestr carries: its byte order and its size in bytes, such as `{"byteorder": "<", "itemsize": 4}` for ` ZarrV2FloatFillValue: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/struct.py index 1640e3e784..f6ebd1b68c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/struct.py @@ -3,44 +3,23 @@ from __future__ import annotations import math -from typing import TYPE_CHECKING, Annotated, Any, Final, cast - -from annotated_types import Ge -from typing_extensions import ReadOnly, TypedDict +from typing import TYPE_CHECKING, Any, Final, cast from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import AcceptedDeclaration -from zarr_metadata.v2._definition import ( - STRUCT_NAME, - ZarrV2DataTypeDeclaration, - ZarrV2DataTypeDefinition, - item_bytes, +from zarr_metadata.v2._definition import STRUCT_NAME, ZarrV2DataTypeDefinition, item_bytes +from zarr_metadata.v2._types.data_type import ( + ZarrV2Base64FillValue, + ZarrV2StructConfiguration, + ZarrV2StructRecord, ) from zarr_metadata.v2.data_type._rules import sized_base64_rules -from zarr_metadata.v2.data_type.fixed_width import ZarrV2Base64FillValue if TYPE_CHECKING: from collections.abc import Iterator from zarr_metadata.definition._definition import Nested -ZarrV2StructRecord = ( - tuple[str, ZarrV2DataTypeDeclaration, tuple[Annotated[int, Ge(0)], ...]] - | tuple[str, ZarrV2DataTypeDeclaration] -) -"""A field record: `[name, dtype]` or `[name, dtype, shape]`. The dtype is a nested declaration (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L152-L174).""" - - -class ZarrV2StructConfiguration(TypedDict, closed=True): - """The records of a structured dtype. The document writes the records as the dtype itself.""" - - fields: ReadOnly[tuple[ZarrV2StructRecord, ...]] - """The field records, in order. - - Their names must be distinct, except `""`, which NumPy gives the padding - of an aligned struct. There may be no records. - """ - def _rules(configuration: ZarrV2StructConfiguration, nested: Nested) -> Iterator[ValidationProblem]: """Check that the names are distinct. The name `""` may repeat: NumPy uses it for the padding of an aligned struct, and zarr-python 2.x writes it. A struct with no records is allowed: NumPy's `np.dtype([])` takes no bytes, and zarr-python 2.x writes it as `[]` and reads it back.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py index ffca4e8662..f499363f43 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py @@ -2,43 +2,22 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated, Final, Literal, NotRequired - -from annotated_types import Ge -from typing_extensions import ReadOnly, TypedDict +from typing import TYPE_CHECKING, Final from zarr_metadata._numpy import ( - NumpyTimeScaleFactor, - NumpyTimeTicks, numpy_time_fill_value_canonical, ) from zarr_metadata.v2._definition import ZarrV2DataTypeDefinition -from zarr_metadata.v2.data_type._rules import time_rules -from zarr_metadata.v2.data_type.scalar import ( - ZarrV2ByteOrder, +from zarr_metadata.v2._types.data_type import ( + ZarrV2TimeConfiguration, + ZarrV2TimeFillValue, ) +from zarr_metadata.v2.data_type._rules import time_rules if TYPE_CHECKING: from zarr_metadata.definition._definition import Nested -class ZarrV2TimeConfiguration(TypedDict, closed=True): - """The configuration that a time typestr carries. For example, ``.""" - itemsize: ReadOnly[Annotated[int, Ge(0)]] - """The size in bytes that the typestr writes, which must be 8.""" - unit: NotRequired[ReadOnly[str]] - """The NumPy time unit in brackets, such as `ns`. The canonical form writes `μs` as `us`. - - An array's dtype must have one. Only a codec's dtype parameter may leave - it out, as in ` ZarrV2TimeConfiguration: """Write `μs` as `us`, as NumPy writes it.""" if configuration.get("unit") == "μs": @@ -46,10 +25,6 @@ def _canonical(configuration: ZarrV2TimeConfiguration) -> ZarrV2TimeConfiguratio return configuration -ZarrV2TimeFillValue = NumpyTimeTicks | Literal["NaT"] | None -"""A v2 time fill value: a count of ticks, `"NaT"`, or null.""" - - def _fill_value_canonical( configuration: ZarrV2TimeConfiguration, nested: Nested, value: ZarrV2TimeFillValue ) -> ZarrV2TimeFillValue: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py index 10f12595c8..5b24dc9f49 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py @@ -31,11 +31,8 @@ from zarr_metadata._renamed import renamed_scope from zarr_metadata.v2._context import ZarrV2Context, ZarrV2ContextSlots, ZarrV2Kind from zarr_metadata.v2._core import CORE_V2_CONTEXT, RENAMED_SCOPES_V2 -from zarr_metadata.v2._definition import ( - ZarrV2CodecDefinition, - ZarrV2DataTypeDeclaration, - ZarrV2DataTypeDefinition, -) +from zarr_metadata.v2._declaration_json import ZarrV2DataTypeDeclaration +from zarr_metadata.v2._definition import ZarrV2CodecDefinition, ZarrV2DataTypeDefinition __all__ = [ "CORE_V2_CONTEXT", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py b/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py index 63bf343873..f647895154 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py @@ -5,14 +5,10 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final +from zarr_metadata._common import EmptyConfiguration from zarr_metadata._typed_json import Schemas, is_alias from zarr_metadata.definition._context import scoped -from zarr_metadata.definition._definition import ( - Definition, - EmptyConfiguration, - as_kind, - declaration_kind, -) +from zarr_metadata.definition._definition import Definition, as_kind, declaration_kind from zarr_metadata.definition._envelope import EXTENSION_NAME_SCHEMA_PATTERN from zarr_metadata.v3._context import ZarrV3Context, ZarrV3Kind, is_v3_kind from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/__init__.py new file mode 100644 index 0000000000..a683fde100 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/__init__.py @@ -0,0 +1,7 @@ +"""The types of the v3 extensions, one module per kind of extension. + +Every module in this package holds types only and imports no definition, +so a definition of any kind can import the types of any other at run +time. Each name is public in the extension's own submodule, which +re-exports it. +""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py new file mode 100644 index 0000000000..ba34cde633 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py @@ -0,0 +1,141 @@ +"""The types of the v3 chunk grids. + +For each chunk grid: the value and `Literal` type of its name, its +configuration and object forms, and the JSON shapes its metadata may take. + +This module holds types only and imports no definition. Each chunk grid's +submodule of `zarr_metadata.v3.chunk_grid` re-exports its names, and +holds the chunk grid's definition and rules. +""" + +from typing import Annotated, Final, Literal, NotRequired + +from annotated_types import Ge +from typing_extensions import TypedDict + +# regular: https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#regular-grids + +REGULAR_CHUNK_GRID_NAME: Final = "regular" +"""The `name` field value of the regular chunk grid.""" + +RegularChunkGridName = Literal["regular"] +"""Literal type of the `name` field of the regular chunk grid.""" + + +class RegularChunkGridConfiguration(TypedDict, closed=True): + """Configuration for the regular chunk grid. + + Every chunk length is at least 1, even along a dimension of length 0: + "Chunk sizes must be greater than zero" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L40). + The core spec's "The chunk shape elements are non-zero when the + corresponding dimensions of the arrays have non-zero length" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L284-L285) + says less about an empty dimension, and it does not allow anything + that the grid spec forbids. + """ + + chunk_shape: tuple[Annotated[int, Ge(1)], ...] + """The length of every chunk along each dimension of the array. + + One entry per dimension, each at least 1. + """ + + +class RegularChunkGridObject(TypedDict, closed=True): + """Regular chunk grid metadata in object form.""" + + name: RegularChunkGridName + """The chunk grid's name, `"regular"`.""" + configuration: RegularChunkGridConfiguration + """The chunk grid's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk grid must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk grid + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ + + +RegularChunkGridMetadata = RegularChunkGridObject +"""Permitted JSON shape for regular chunk grid metadata. + +`chunk_shape` is required and has no default, so only the object form is +valid; the short-hand-name form is not permitted by the spec for this grid. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L528-L537 ("must be an object with the names name and configuration") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 +""" + + +# rectilinear: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md + +RECTILINEAR_CHUNK_GRID_NAME: Final = "rectilinear" +"""The `name` field value of the rectilinear chunk grid.""" + +RectilinearChunkGridName = Literal["rectilinear"] +"""Literal type of the `name` field of the rectilinear chunk grid.""" + +_Positive = Annotated[int, Ge(1)] + +RectilinearDimSpec = _Positive | tuple[_Positive | tuple[_Positive, _Positive], ...] +"""JSON shape for one dimension's rectilinear spec. + +Either a bare integer (uniform shorthand for a regular dimension within +a rectilinear grid), or a tuple of integers and/or `[value, count]` RLE +pairs. Every extent, and every run's length and count, is at least 1. +""" + + +class RectilinearChunkGridConfiguration(TypedDict, closed=True): + """Configuration for the rectilinear chunk grid.""" + + kind: Literal["inline"] + """How `chunk_shapes` is encoded; `"inline"` is the only kind the spec defines.""" + chunk_shapes: tuple[RectilinearDimSpec, ...] + """The chunk edge lengths along each dimension of the array, one entry per dimension. + + An entry is a bare integer, a step repeated until it covers the + dimension, or a list of edge lengths and `[length, count]` runs, whose + lengths must sum to at least the dimension's length. Every length and + count is at least 1. + """ + + +class RectilinearChunkGridObject(TypedDict, closed=True): + """Rectilinear chunk grid metadata in object form.""" + + name: RectilinearChunkGridName + """The chunk grid's name, `"rectilinear"`.""" + configuration: RectilinearChunkGridConfiguration + """The chunk grid's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk grid must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk grid + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ + + +RectilinearChunkGridMetadata = RectilinearChunkGridObject +"""Permitted JSON shape for rectilinear chunk grid metadata. + +`kind` and `chunk_shapes` are required, so only the object form is valid; +the short-hand-name form is not permitted by the spec for this grid. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md#L59-L62 + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 +""" + + +__all__ = [ + "RECTILINEAR_CHUNK_GRID_NAME", + "REGULAR_CHUNK_GRID_NAME", + "RectilinearChunkGridConfiguration", + "RectilinearChunkGridMetadata", + "RectilinearChunkGridName", + "RectilinearChunkGridObject", + "RectilinearDimSpec", + "RegularChunkGridConfiguration", + "RegularChunkGridMetadata", + "RegularChunkGridName", + "RegularChunkGridObject", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_key_encoding.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_key_encoding.py new file mode 100644 index 0000000000..f73b73f0cc --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_key_encoding.py @@ -0,0 +1,147 @@ +"""The types of the v3 chunk key encodings. + +For each chunk key encoding: the value and `Literal` type of its name, +its configuration and object forms, the JSON shapes its metadata may +take, and the constants that spell its permitted separators. + +This module holds types only and imports no definition. Each chunk key +encoding's submodule of `zarr_metadata.v3.chunk_key_encoding` re-exports +its names, and holds the chunk key encoding's definition. +""" + +from typing import Final, Literal, NotRequired + +from typing_extensions import TypedDict + +# default: https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding + +DEFAULT_CHUNK_KEY_ENCODING_NAME: Final = "default" +"""The `name` field value of the default chunk key encoding.""" + +DefaultChunkKeyEncodingName = Literal["default"] +"""Literal type of the `name` field of the default chunk key encoding.""" + +DefaultChunkKeyEncodingSeparator = Literal["/", "."] +"""Literal type of permitted `separator` values for the default chunk key encoding. + +Defaults to `"/"` if absent. +""" + +DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR: Final = ("/", ".") +"""Tuple of permitted values for the `separator` field of the default chunk key encoding.""" + + +class DefaultChunkKeyEncodingConfiguration(TypedDict, closed=True): + """Configuration for the default chunk key encoding. + + `separator` is optional and defaults to `"/"` per spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst#L27-L29 + """ + + separator: NotRequired[DefaultChunkKeyEncodingSeparator] + """The character between the parts of a chunk key: `c/1/23/45` or `c.1.23.45`. + + Absent means `"/"`. + """ + + +class DefaultChunkKeyEncodingObject(TypedDict, closed=True): + """Default chunk key encoding metadata in object form.""" + + name: DefaultChunkKeyEncodingName + """The chunk key encoding's name, `"default"`.""" + configuration: NotRequired[DefaultChunkKeyEncodingConfiguration] + """The chunk key encoding's parameters. + + Absent means every parameter takes its default: a `separator` of `"/"`. + """ + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk key encoding must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk key encoding + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ + + +DefaultChunkKeyEncodingMetadata = DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName +"""Permitted JSON shapes for the default chunk-key encoding metadata. + +The configuration has no required keys (`separator` defaults to `"/"`), +so the short-hand-name form is permitted in addition to the object form. +""" + + +# v2: https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding + +V2_CHUNK_KEY_ENCODING_NAME: Final = "v2" +"""The `name` field value of the v2 chunk key encoding.""" + +V2ChunkKeyEncodingName = Literal["v2"] +"""Literal type of the `name` field of the v2 chunk key encoding.""" + +V2ChunkKeyEncodingSeparator = Literal["/", "."] +"""Literal type of permitted `separator` values for the v2 chunk key encoding. + +Defaults to `"."` if absent. +""" + +V2_CHUNK_KEY_ENCODING_SEPARATOR: Final = ("/", ".") +"""Tuple of permitted values for the `separator` field of the v2 chunk key encoding.""" + + +class V2ChunkKeyEncodingConfiguration(TypedDict, closed=True): + """Configuration for the v2 chunk key encoding. + + `separator` is optional and defaults to `"."` per spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst#L27-L29 + """ + + separator: NotRequired[V2ChunkKeyEncodingSeparator] + """The character between the parts of a chunk key: `1.23.45` or `1/23/45`. + + Absent means `"."`. + """ + + +class V2ChunkKeyEncodingObject(TypedDict, closed=True): + """v2-compatibility chunk key encoding metadata in object form.""" + + name: V2ChunkKeyEncodingName + """The chunk key encoding's name, `"v2"`.""" + configuration: NotRequired[V2ChunkKeyEncodingConfiguration] + """The chunk key encoding's parameters. + + Absent means every parameter takes its default: a `separator` of `"."`. + """ + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk key encoding must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk key encoding + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ + + +V2ChunkKeyEncodingMetadata = V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName +"""Permitted JSON shapes for the v2-compatibility chunk-key encoding metadata. + +The configuration has no required keys (`separator` defaults to `"."`), +so the short-hand-name form is permitted in addition to the object form. +""" + + +__all__ = [ + "DEFAULT_CHUNK_KEY_ENCODING_NAME", + "DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR", + "V2_CHUNK_KEY_ENCODING_NAME", + "V2_CHUNK_KEY_ENCODING_SEPARATOR", + "DefaultChunkKeyEncodingConfiguration", + "DefaultChunkKeyEncodingMetadata", + "DefaultChunkKeyEncodingName", + "DefaultChunkKeyEncodingObject", + "DefaultChunkKeyEncodingSeparator", + "V2ChunkKeyEncodingConfiguration", + "V2ChunkKeyEncodingMetadata", + "V2ChunkKeyEncodingName", + "V2ChunkKeyEncodingObject", + "V2ChunkKeyEncodingSeparator", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py new file mode 100644 index 0000000000..00792264ea --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py @@ -0,0 +1,809 @@ +"""The types of the v3 codecs. + +For each codec: the value and `Literal` type of its name, its +configuration and object forms, the JSON shapes its metadata may take, +and the constants that spell its permitted values. + +This module holds types only and imports no definition, so a definition +of any kind can import a codec's types at run time. Each codec's +submodule of `zarr_metadata.v3.codec` re-exports its names, and holds the +codec's definition and rules. +""" + +from typing import Annotated, Final, Literal, NotRequired + +from annotated_types import Ge, Interval +from typing_extensions import TypedDict + +from zarr_metadata._common import EmptyConfiguration, JSONValue +from zarr_metadata.v3._common import ( + ZarrV3CodecDeclaration, + ZarrV3DataTypeDeclaration, + ZarrV3StaticCodecDeclaration, +) + +# blosc: https://zarr-specs.readthedocs.io/en/latest/v3/codecs/blosc/index.html + +BLOSC_CODEC_NAME: Final = "blosc" +"""The `name` field value of the `blosc` codec.""" + +BloscCodecName = Literal["blosc"] +"""Literal type of the `name` field of the `blosc` codec.""" + +BloscShuffle = Literal["noshuffle", "shuffle", "bitshuffle"] +"""Literal type of blosc shuffle mode names.""" + +BLOSC_SHUFFLE: Final = ("noshuffle", "shuffle", "bitshuffle") +"""Tuple of permitted values for the `shuffle` field of the `blosc` codec.""" + +BloscCName = Literal["lz4", "lz4hc", "blosclz", "snappy", "zlib", "zstd"] +"""Literal type of blosc compressor identifiers.""" + +BLOSC_CNAME: Final = ("lz4", "lz4hc", "blosclz", "snappy", "zlib", "zstd") +"""Tuple of permitted values for the `cname` field of the `blosc` codec.""" + + +class BloscCodecConfiguration(TypedDict, closed=True): + """Configuration for the Zarr v3 `blosc` codec.""" + + cname: BloscCName + """The compressor blosc applies to each block, one of `BLOSC_CNAME`.""" + clevel: Annotated[int, Interval(ge=0, le=9)] + """The compression level, an integer from 0 to 9. + + 0 turns compression off; 1 is the fastest and compresses least, 9 the + slowest and compresses most. + """ + shuffle: BloscShuffle + """The shuffling applied before compression. + + `"noshuffle"` for none, `"shuffle"` for byte-wise, `"bitshuffle"` for + bit-wise. + """ + blocksize: Annotated[int, Ge(0)] + """The size in bytes of the blocks the buffer is divided into before compression. + + At least 0; 0 lets blosc choose the size. + """ + typesize: NotRequired[int] + """The stride in bytes over which shuffling is performed. + + Required, and at least 1, unless `shuffle` is `"noshuffle"`, under + which the spec ignores it and `canonicalize` drops it. + """ + + +class BloscCodecObject(TypedDict, closed=True): + """`blosc` codec metadata in object form.""" + + name: BloscCodecName + """The codec's name, `"blosc"`.""" + configuration: BloscCodecConfiguration + """The codec's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +BloscCodecMetadata = BloscCodecObject +"""Permitted JSON shape for `blosc` codec metadata. + +The configuration has multiple required keys (`cname`, `clevel`, `shuffle`, +`blocksize`), so only the object form is valid; the short-hand-name form +is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/blosc/index.rst#L57-L98 (configuration parameters) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") +""" + +BLOSC_NO_SHUFFLE: Final = "noshuffle" +"""The `shuffle` value under which `typesize` carries no information. + +The spec requires `typesize` "unless `shuffle` is `"noshuffle"`, in which +case the value is ignored", so this is the one value that changes whether +another member is required. +""" + + +# bytes: https://zarr-specs.readthedocs.io/en/latest/v3/codecs/bytes/index.html + +BYTES_CODEC_NAME: Final = "bytes" +"""The `name` field value of the `bytes` codec.""" + +BytesCodecName = Literal["bytes"] +"""Literal type of the `name` field of the `bytes` codec.""" + +Endianness = Literal["little", "big"] +"""Literal type of byte order of multi-byte numeric data.""" + +ENDIANNESS: Final = ("little", "big") +"""Tuple of permitted values for the `endian` field of the `bytes` codec.""" + + +class BytesCodecConfiguration(TypedDict, closed=True): + """ + Configuration for the Zarr v3 `bytes` codec. + + The `endian` field is required for multi-byte data types. + """ + + endian: NotRequired[Endianness] + """The byte order in which each value is written. + + Required when the values of the data type the codec is handed take more + than one byte; a single-byte type such as `uint8` or `bool` does not + depend on it. + """ + + +class BytesCodecObject(TypedDict, closed=True): + """`bytes` codec metadata in object form. + + `configuration` is itself optional — when no configuration fields are + set, the entire `configuration` key may be omitted. This matches the + bare-string short-hand form (`BytesCodecName`) at the canonical data + level; both encodings describe a `bytes` codec with default settings. + """ + + name: BytesCodecName + """The codec's name, `"bytes"`.""" + configuration: NotRequired[BytesCodecConfiguration] + """The codec's parameters. + + May be absent when no parameter is set. + """ + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +BytesCodecMetadata = BytesCodecObject | BytesCodecName +"""Permitted JSON shapes for `bytes` codec metadata. + +The configuration has no required keys (`endian` is conditionally required +at runtime based on data type), so the spec's short-hand-name form is +permitted in addition to the object form, and the object form may itself +omit `configuration` entirely. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69 ("endian: Required for data types for which endianness is applicable") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 +""" + + +# cast_value: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md + +CAST_VALUE_CODEC_NAME: Final = "cast_value" +"""The `name` field value of the `cast_value` codec.""" + +CastValueCodecName = Literal["cast_value"] +"""Literal type of the `name` field of the `cast_value` codec.""" + +CastRoundingMode = Literal[ + "nearest-even", + "towards-zero", + "towards-positive", + "towards-negative", + "nearest-away", +] +"""Literal type of permitted values for the `rounding` configuration field. + +Defaults to `"nearest-even"` if absent. +""" + +CAST_ROUNDING_MODE: Final = ( + "nearest-even", + "towards-zero", + "towards-positive", + "towards-negative", + "nearest-away", +) +"""Tuple of permitted values for the `rounding` field of the `cast_value` codec.""" + +CastOutOfRangeMode = Literal["clamp", "wrap"] +"""Literal type of permitted values for the `out_of_range` configuration field. + +If absent, out-of-range values are an encoding/decoding error. +""" + +CAST_OUT_OF_RANGE_MODE: Final = ("clamp", "wrap") +"""Tuple of permitted values for the `out_of_range` field of the `cast_value` codec.""" + +ScalarMapEntry = tuple[JSONValue, JSONValue] +"""A single `[input, output]` mapping in a `scalar_map` direction. + +Each scalar is encoded as a fill value of its data type. For a float, +the infinities are `"Infinity"` and `"-Infinity"`, as the core encoding +writes them. The codec's own example writes `"+Infinity"` instead. +""" + + +class ScalarMap(TypedDict, closed=True): + """Optional encode/decode scalar overrides for the cast_value codec.""" + + encode: NotRequired[tuple[ScalarMapEntry, ...]] + """The `[input, output]` pairs applied when encoding: input in the array's data type, output in `data_type`. + + Each scalar is written in its data type's fill value encoding. Where an + input repeats, the first pair applies. + """ + decode: NotRequired[tuple[ScalarMapEntry, ...]] + """The `[input, output]` pairs applied when decoding: input in `data_type`, output in the array's data type. + + Each scalar is written in its data type's fill value encoding. Where an + input repeats, the first pair applies. + """ + + +class CastValueCodecConfiguration(TypedDict, closed=True): + """ + Configuration for the Zarr v3 `cast_value` codec. + + `data_type` is the target data type that input values are cast to. It + is the same shape as the top-level array `data_type` field: either a + bare-string primitive name or a `{name, configuration}` envelope. + """ + + data_type: ZarrV3DataTypeDeclaration + """The data type values are cast to when encoding, and the one the decoder is handed. + + The codec is defined only for data types that model real numbers, the + integer and floating-point types: a boolean, raw-bits, string, bytes, + struct or complex type is refused, here and as the data type the codec + is handed. + """ + rounding: NotRequired[CastRoundingMode] + """How a value `data_type` cannot represent exactly is rounded. + + Absent means `"nearest-even"`. + """ + out_of_range: NotRequired[CastOutOfRangeMode] + """How a value outside the range of `data_type` is brought into it, after rounding. + + `"clamp"` takes the nearest bound; `"wrap"` takes the value congruent + modulo 2^N, N being the width in bits of `data_type`, and is permitted + only when `data_type` is an integral type. Absent means an out-of-range + value is an encoding or decoding error. + """ + scalar_map: NotRequired[ScalarMap] + """Scalars mapped explicitly, in place of `rounding` and `out_of_range`.""" + + +class CastValueCodecObject(TypedDict, closed=True): + """`cast_value` codec metadata in object form.""" + + name: CastValueCodecName + """The codec's name, `"cast_value"`.""" + configuration: CastValueCodecConfiguration + """The codec's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +CastValueCodecMetadata = CastValueCodecObject +"""Permitted JSON shape for `cast_value` codec metadata. + +`configuration.data_type` is required, so only the object form is valid; +the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md#L33-L36 and #L46-L48 (required fields) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") +""" + + +# crc32c: https://zarr-specs.readthedocs.io/en/latest/v3/codecs/crc32c/index.html + +CRC32C_CODEC_NAME: Final = "crc32c" +"""The `name` field value of the `crc32c` codec.""" + +Crc32cCodecName = Literal["crc32c"] +"""Literal type of the `name` field of the `crc32c` codec.""" + + +class Crc32cCodecObject(TypedDict, closed=True): + """`crc32c` codec metadata in object form. + + Per spec the codec has no configuration fields. `configuration` is + optional and, if present, should be an empty mapping. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/crc32c/index.rst#L63-L66 + """ + + name: Crc32cCodecName + """The codec's name, `"crc32c"`.""" + configuration: NotRequired[EmptyConfiguration] + """The codec's parameters, of which there are none. + + Absent, or an empty mapping. + """ + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +Crc32cCodecMetadata = Crc32cCodecObject | Crc32cCodecName +"""Permitted JSON shapes for `crc32c` codec metadata. + +The spec's Extension definition allows extensions with no required +configuration to be encoded as a bare short-hand name. CRC32C has no +configuration, so both forms are valid. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 +""" + + +# gzip: https://zarr-specs.readthedocs.io/en/latest/v3/codecs/gzip/index.html + +GZIP_CODEC_NAME: Final = "gzip" +"""The `name` field value of the `gzip` codec.""" + +GzipCodecName = Literal["gzip"] +"""Literal type of the `name` field of the `gzip` codec.""" + + +class GzipCodecConfiguration(TypedDict, closed=True): + """ + Configuration for the Zarr v3 `gzip` codec. + + `level` is an integer in the range 0-9; 0 disables compression and 9 + is slowest with the best compression ratio. The codec's compressed + output depends on `level`, so metadata that omits it cannot + reproducibly identify the chunk bytes produced by a writer — `level` + is required for the metadata to fulfill its reproducibility role, + even though the spec text does not mark it required with RFC 2119 + keywords. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/gzip/index.rst#L57-L66 + """ + + level: Annotated[int, Interval(ge=0, le=9)] + """The compression level, an integer from 0 to 9. + + 0 turns compression off; 1 is the fastest and compresses least, 9 the + slowest and compresses most. + """ + + +class GzipCodecObject(TypedDict, closed=True): + """`gzip` codec metadata in object form.""" + + name: GzipCodecName + """The codec's name, `"gzip"`.""" + configuration: GzipCodecConfiguration + """The codec's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +GzipCodecMetadata = GzipCodecObject +"""Permitted JSON shape for `gzip` codec metadata. + +`configuration.level` is required (it determines the codec's output bytes +and is therefore part of the metadata's reproducibility contract), so +only the object form is valid; the short-hand-name form is not permitted. +""" + + +# scale_offset: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md + +SCALE_OFFSET_CODEC_NAME: Final = "scale_offset" +"""The `name` field value of the `scale_offset` codec.""" + +ScaleOffsetCodecName = Literal["scale_offset"] +"""Literal type of the `name` field of the `scale_offset` codec.""" + + +class ScaleOffsetCodecConfiguration(TypedDict, closed=True): + """ + Configuration for the Zarr v3 `scale_offset` codec. + + Both fields are optional. A missing `offset` is the additive identity + (e.g. 0 for numeric types); a missing `scale` is the multiplicative + identity (e.g. 1). Each scalar is encoded as a fill value of the input + data type of the codec: an integer for an integer type, and for a + float type a number, `"NaN"`, `"Infinity"`, `"-Infinity"` or a hex + string. + """ + + offset: NotRequired[JSONValue] + """The quantity subtracted from each value when encoding. + + Absent means the additive identity, e.g. 0. A fill value of the data + type the codec is handed. + """ + scale: NotRequired[JSONValue] + """The quantity each value is multiplied by when encoding, after `offset` is subtracted. + + Absent means the multiplicative identity, e.g. 1. A fill value of the + data type the codec is handed. + """ + + +class ScaleOffsetCodecObject(TypedDict, closed=True): + """`scale_offset` codec metadata in object form. + + `configuration` is itself optional per spec — when both `offset` and + `scale` are at their identity defaults, the codec is a no-op and the + entire `configuration` field may be omitted. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md#L18 and #L35 + """ + + name: ScaleOffsetCodecName + """The codec's name, `"scale_offset"`.""" + configuration: NotRequired[ScaleOffsetCodecConfiguration] + """The codec's parameters. + + May be absent when neither `offset` nor `scale` is set, which makes the + codec a no-op. + """ + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +ScaleOffsetCodecMetadata = ScaleOffsetCodecObject | ScaleOffsetCodecName +"""Permitted JSON shapes for `scale_offset` codec metadata. + +The configuration has no required keys (both `offset` and `scale` are +optional, and the configuration itself is optional), so the short-hand-name +form is permitted in addition to the object form. +""" + + +# sharding_indexed: https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/index.html + +SHARDING_INDEXED_CODEC_NAME: Final = "sharding_indexed" +"""The `name` field value of the `sharding_indexed` codec.""" + +ShardingIndexedCodecName = Literal["sharding_indexed"] +"""Literal type of the `name` field of the `sharding_indexed` codec.""" + +ShardingIndexLocation = Literal["start", "end"] +"""Literal type of the position of the shard index within the encoded shard.""" + +SHARDING_INDEX_LOCATION: Final = ("start", "end") +"""Tuple of permitted values for the `index_location` field of the `sharding_indexed` codec.""" + + +class ShardingIndexedCodecConfiguration(TypedDict, closed=True): + """ + Configuration for the Zarr v3 `sharding_indexed` codec. + + `chunk_shape` is the shape of inner chunks along each dimension; + it must evenly divide the shard shape. + + `codecs` is the codec pipeline applied to each inner chunk; exactly + one array-to-bytes codec is required. + + `index_codecs` is the codec pipeline applied to the shard index. It may + contain only codecs of static size, because a reader locates the index + by a size that it knows before reading it. A compressor in this + pipeline is therefore rejected. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L147-L155 + + `index_location` defaults to `"end"` per the spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L157-L161 + """ + + chunk_shape: tuple[Annotated[int, Ge(1)], ...] + """The shape of the inner chunks a shard is divided into. + + One length, at least 1, per axis of the shard the codec is handed, each + dividing the shard's length along that axis. + """ + codecs: tuple[ZarrV3CodecDeclaration, ...] + """The codec pipeline that encodes each inner chunk. + + Like an array's `codecs`, it holds exactly one array -> bytes codec. + """ + index_codecs: tuple[ZarrV3StaticCodecDeclaration, ...] + """The codec pipeline that encodes the shard index. + + It holds exactly one array -> bytes codec, and only codecs whose output + size is fixed by the size of their input, so no compressor. The spec + recommends a little-endian `bytes` codec followed by `crc32c`. + """ + index_location: NotRequired[ShardingIndexLocation] + """Whether the shard index sits at the `"start"` or the `"end"` of the encoded shard. + + Absent means `"end"`. + """ + + +class ShardingIndexedCodecObject(TypedDict, closed=True): + """`sharding_indexed` codec metadata in object form.""" + + name: ShardingIndexedCodecName + """The codec's name, `"sharding_indexed"`.""" + configuration: ShardingIndexedCodecConfiguration + """The codec's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +ShardingIndexedCodecMetadata = ShardingIndexedCodecObject +"""Permitted JSON shape for `sharding_indexed` codec metadata. + +The configuration has multiple required keys (`chunk_shape`, `codecs`, +`index_codecs`), so only the object form is valid; the short-hand-name +form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L129-L138 (`chunk_shape`) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L141-L155 (`codecs` and `index_codecs`, the members the spec marks required) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") +""" + + +# transpose: https://zarr-specs.readthedocs.io/en/latest/v3/codecs/transpose/index.html + +TRANSPOSE_CODEC_NAME: Final = "transpose" +"""The `name` field value of the `transpose` codec.""" + +TransposeCodecName = Literal["transpose"] +"""Literal type of the `name` field of the `transpose` codec.""" + + +class TransposeCodecConfiguration(TypedDict, closed=True): + """ + Configuration for the Zarr v3 `transpose` codec. + + `order` is a permutation of the dimension indices 0..n-1 that + specifies the dimension reordering applied during encoding. + """ + + order: tuple[int, ...] + """The reordering of axes applied when encoding: axis `i` of the encoded chunk is axis `order[i]` of the chunk the codec is handed. + + A permutation of `0, 1, ..., n-1`, `n` being the number of axes of the + chunk the codec is handed. + """ + + +class TransposeCodecObject(TypedDict, closed=True): + """`transpose` codec metadata in object form.""" + + name: TransposeCodecName + """The codec's name, `"transpose"`.""" + configuration: TransposeCodecConfiguration + """The codec's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +TransposeCodecMetadata = TransposeCodecObject +"""Permitted JSON shape for `transpose` codec metadata. + +`order` is required, so only the object form is valid; the short-hand-name +form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/transpose/index.rst#L60-L66 ("order: Required") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") +""" + + +# vlen_bytes: https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/codecs/vlen-bytes/README.md + +VLEN_BYTES_CODEC_NAME: Final = "vlen-bytes" +"""The `name` field value of the `vlen-bytes` codec.""" + +VlenBytesCodecName = Literal["vlen-bytes"] +"""Literal type of the `name` field of the `vlen-bytes` codec.""" + + +class VlenBytesCodecObject(TypedDict, closed=True): + """`vlen-bytes` codec metadata in object form. The codec has no configuration fields.""" + + name: VlenBytesCodecName + """The codec's name, `"vlen-bytes"`.""" + configuration: NotRequired[EmptyConfiguration] + """The codec's parameters, of which there are none. + + Absent, or an empty mapping. + """ + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +VlenBytesCodecMetadata = VlenBytesCodecObject | VlenBytesCodecName +"""Permitted JSON shapes for `vlen-bytes` codec metadata: the object form, or the bare name, since the codec has no configuration.""" + + +# vlen_utf8: https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/codecs/vlen-utf8/README.md + +VLEN_UTF8_CODEC_NAME: Final = "vlen-utf8" +"""The `name` field value of the `vlen-utf8` codec.""" + +VlenUtf8CodecName = Literal["vlen-utf8"] +"""Literal type of the `name` field of the `vlen-utf8` codec.""" + + +class VlenUtf8CodecObject(TypedDict, closed=True): + """`vlen-utf8` codec metadata in object form. The codec has no configuration fields.""" + + name: VlenUtf8CodecName + """The codec's name, `"vlen-utf8"`.""" + configuration: NotRequired[EmptyConfiguration] + """The codec's parameters, of which there are none. + + Absent, or an empty mapping. + """ + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +VlenUtf8CodecMetadata = VlenUtf8CodecObject | VlenUtf8CodecName +"""Permitted JSON shapes for `vlen-utf8` codec metadata: the object form, or the bare name, since the codec has no configuration.""" + + +# zstd: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md + +ZSTD_CODEC_NAME: Final = "zstd" +"""The `name` field value of the `zstd` codec.""" + +ZstdCodecName = Literal["zstd"] +"""Literal type of the `name` field of the `zstd` codec.""" + +ZSTD_MIN_LEVEL: Final = -131072 +"""The lowest `level` zstd accepts: ZSTD_minCLevel(), -(1 << 17).""" + +ZSTD_MAX_LEVEL: Final = 22 +"""The highest `level` zstd accepts: ZSTD_maxCLevel().""" + + +class ZstdCodecConfiguration(TypedDict, closed=True): + """ + Configuration for the Zarr v3 `zstd` codec. + + `level` is required; `checksum` is optional ("Should be omitted if + false"). + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 + """ + + level: Annotated[int, Interval(ge=ZSTD_MIN_LEVEL, le=ZSTD_MAX_LEVEL)] + """The compression level, an integer from `ZSTD_MIN_LEVEL` (-131072) to `ZSTD_MAX_LEVEL` (22). + + 0 means zstd's default level; otherwise a higher level compresses more, + more slowly. It has no effect on decoding. + """ + checksum: NotRequired[bool] + """Whether a checksum is stored when writing and verified when reading. + + Absent means `false`; the registry says to omit it rather than write + `false`. + """ + + +class ZstdCodecObject(TypedDict, closed=True): + """`zstd` codec metadata in object form.""" + + name: ZstdCodecName + """The codec's name, `"zstd"`.""" + configuration: ZstdCodecConfiguration + """The codec's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ + + +ZstdCodecMetadata = ZstdCodecObject +"""Permitted JSON shape for `zstd` codec metadata. + +`level` is required, so only the object form is valid; the short-hand-name +form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") +""" + + +__all__ = [ + "BLOSC_CNAME", + "BLOSC_CODEC_NAME", + "BLOSC_NO_SHUFFLE", + "BLOSC_SHUFFLE", + "BYTES_CODEC_NAME", + "CAST_OUT_OF_RANGE_MODE", + "CAST_ROUNDING_MODE", + "CAST_VALUE_CODEC_NAME", + "CRC32C_CODEC_NAME", + "ENDIANNESS", + "GZIP_CODEC_NAME", + "SCALE_OFFSET_CODEC_NAME", + "SHARDING_INDEXED_CODEC_NAME", + "SHARDING_INDEX_LOCATION", + "TRANSPOSE_CODEC_NAME", + "VLEN_BYTES_CODEC_NAME", + "VLEN_UTF8_CODEC_NAME", + "ZSTD_CODEC_NAME", + "ZSTD_MAX_LEVEL", + "ZSTD_MIN_LEVEL", + "BloscCName", + "BloscCodecConfiguration", + "BloscCodecMetadata", + "BloscCodecName", + "BloscCodecObject", + "BloscShuffle", + "BytesCodecConfiguration", + "BytesCodecMetadata", + "BytesCodecName", + "BytesCodecObject", + "CastOutOfRangeMode", + "CastRoundingMode", + "CastValueCodecConfiguration", + "CastValueCodecMetadata", + "CastValueCodecName", + "CastValueCodecObject", + "Crc32cCodecMetadata", + "Crc32cCodecName", + "Crc32cCodecObject", + "Endianness", + "GzipCodecConfiguration", + "GzipCodecMetadata", + "GzipCodecName", + "GzipCodecObject", + "ScalarMap", + "ScalarMapEntry", + "ScaleOffsetCodecConfiguration", + "ScaleOffsetCodecMetadata", + "ScaleOffsetCodecName", + "ScaleOffsetCodecObject", + "ShardingIndexLocation", + "ShardingIndexedCodecConfiguration", + "ShardingIndexedCodecMetadata", + "ShardingIndexedCodecName", + "ShardingIndexedCodecObject", + "TransposeCodecConfiguration", + "TransposeCodecMetadata", + "TransposeCodecName", + "TransposeCodecObject", + "VlenBytesCodecMetadata", + "VlenBytesCodecName", + "VlenBytesCodecObject", + "VlenUtf8CodecMetadata", + "VlenUtf8CodecName", + "VlenUtf8CodecObject", + "ZstdCodecConfiguration", + "ZstdCodecMetadata", + "ZstdCodecName", + "ZstdCodecObject", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/consolidated.py new file mode 100644 index 0000000000..e321f48fec --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/consolidated.py @@ -0,0 +1,83 @@ +"""The types of v3 consolidated metadata: the envelope that the core spec fixes, and the entries as the reference implementation writes them. + +This module holds types only and imports no definition. +`zarr_metadata.v3.consolidated` re-exports its names, and holds the +definition that reads a group's `consolidated_metadata` member. +""" + +from collections.abc import Mapping +from typing import Final, Literal + +from typing_extensions import TypedDict + +from zarr_metadata._common import UserJSONValue +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON + + +class ZarrV3ConsolidatedMetadataEnvelopeJSON(TypedDict, closed=True): + """ + The `consolidated_metadata` member of a v3 group document, as the core spec fixes it. + + The spec allows this one additional field on a group, for historical + reasons, and fixes its envelope: `kind` is `"inline"`, + `must_understand` is false, and `metadata` is an object. The spec + leaves the entries of `metadata` to the reference implementation, so + here they are any JSON, as user data: the attributes of a listed + document may hold `NaN`, `Infinity` and `-Infinity`. + `ZarrV3ConsolidatedMetadataJSON` types the + entries as the reference implementation writes them: an array or group + document at each path. + + See https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 + """ + + kind: Literal["inline"] + """How the entries are carried, `"inline"`: in the group's own document.""" + must_understand: Literal[False] + """Always `false`: a reader that does not recognize consolidated metadata may ignore it.""" + metadata: Mapping[str, UserJSONValue] + """The entries, by each node's path from the group, as any JSON.""" + + +class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): + """ + Inline consolidated metadata embedded in a v3 group. + + The `metadata` map contains only v3 array and group entries. V2 entries + are excluded from this interoperability convention by design. The v3 core + specification acknowledges `consolidated_metadata` as a historical + additional field and fixes this envelope, but leaves the entries to the + reference implementation: + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 + """ + + kind: Literal["inline"] + """How the entries are carried, `"inline"`: in the group's own document.""" + must_understand: Literal[False] + """Always `false`: a reader that does not recognize consolidated metadata may ignore it.""" + metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] + """The metadata document of each node below the group, by its path from the group. + + A path joins its segments with `"/"`, e.g. `"child/grandchild"`. + """ + + +ZARR_V3_CONSOLIDATED_METADATA_KEY: Final = "consolidated_metadata" +"""The key under which consolidated metadata is embedded in a v3 group document. + +Unlike the v2 `.zmetadata` file, this is not a store key: consolidated metadata +is carried as an additional field inside the group's own `zarr.json`. The core +spec names the field and its envelope ("For historical reasons, group metadata +documents may contain an additional field named ``consolidated_metadata``"); +the entry format, like the v2 counterpart, is a reference-implementation +convention. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 +""" + + +__all__ = [ + "ZARR_V3_CONSOLIDATED_METADATA_KEY", + "ZarrV3ConsolidatedMetadataEnvelopeJSON", + "ZarrV3ConsolidatedMetadataJSON", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py new file mode 100644 index 0000000000..3c8b19d703 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py @@ -0,0 +1,768 @@ +"""The types of the v3 data types. + +For each data type: the value and `Literal` type of its name, its +configuration and object forms where it has them, the JSON shape of its +fill value, and the constants that spell its permitted values. + +This module holds types only and imports no definition, so a definition +of any kind can import a data type's types at run time. Each data type's +submodule of `zarr_metadata.v3.data_type` re-exports its names, and holds +the data type's definition and rules. +""" + +from collections.abc import Mapping +from typing import Annotated, Final, Literal, NewType, NotRequired, get_args + +from annotated_types import Interval +from typing_extensions import ReadOnly, TypedDict + +from zarr_metadata._common import JSONValue +from zarr_metadata._numpy import NUMPY_TIME_UNITS, Base64Bytes, NumpyTimeScaleFactor, NumpyTimeTicks +from zarr_metadata.v3._common import ZarrV3DataTypeDeclaration + +# Shared by more than one data type + +ByteValue = Annotated[int, Interval(ge=0, le=255)] +"""One byte of a fill value: a JSON integer in `[0, 255]`, as the fill values of raw bits and of `bytes` write each byte. + +See https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L97-L99 +for raw bits, and https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md?plain=1#L8 +for `bytes`. +""" + +FloatSpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] +"""The named non-finite fill values every IEEE 754 floating-point type takes.""" + +FLOAT_SPECIAL_FILL_VALUES: Final[tuple[FloatSpecialFillValue, ...]] = get_args( + FloatSpecialFillValue +) +"""The strings that the fill value rules accept for a float fill value in place of a number.""" + +NumpyTimeUnit = Literal[ + "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" +] +"""The time unit codes of the numpy time types, `numpy.datetime64` and `numpy.timedelta64`.""" + +NUMPY_TIME_UNIT: Final = NUMPY_TIME_UNITS +"""Runtime tuple of the permitted `numpy.timedelta64`/`numpy.datetime64` unit strings.""" + + +# bool: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +BOOL_DATA_TYPE_NAME: Final = "bool" +"""The `data_type` value for the `bool` type.""" + +BoolDataTypeName = Literal["bool"] +"""Literal type of the `data_type` field for `bool`.""" + +BoolFillValue = bool +"""Permitted JSON shape of the `fill_value` field for `bool`: a JSON boolean.""" + + +# int8: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +INT8_DATA_TYPE_NAME: Final = "int8" +"""The `data_type` value for the `int8` type.""" + +Int8DataTypeName = Literal["int8"] +"""Literal type of the `data_type` field for `int8`.""" + +Int8FillValue = Annotated[int, Interval(ge=-(2**7), le=2**7 - 1)] +"""Permitted JSON shape of the `fill_value` field for `int8`: a JSON integer in [-128, 127].""" + + +# int16: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +INT16_DATA_TYPE_NAME: Final = "int16" +"""The `data_type` value for the `int16` type.""" + +Int16DataTypeName = Literal["int16"] +"""Literal type of the `data_type` field for `int16`.""" + +Int16FillValue = Annotated[int, Interval(ge=-(2**15), le=2**15 - 1)] +"""Permitted JSON shape of the `fill_value` field for `int16`: a JSON integer in [-32768, 32767].""" + + +# int32: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +INT32_DATA_TYPE_NAME: Final = "int32" +"""The `data_type` value for the `int32` type.""" + +Int32DataTypeName = Literal["int32"] +"""Literal type of the `data_type` field for `int32`.""" + +Int32FillValue = Annotated[int, Interval(ge=-(2**31), le=2**31 - 1)] +"""Permitted JSON shape of the `fill_value` field for `int32`: a JSON integer in [-2**31, 2**31 - 1].""" + + +# int64: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +INT64_DATA_TYPE_NAME: Final = "int64" +"""The `data_type` value for the `int64` type.""" + +Int64DataTypeName = Literal["int64"] +"""Literal type of the `data_type` field for `int64`.""" + +Int64FillValue = Annotated[int, Interval(ge=-(2**63), le=2**63 - 1)] +"""Permitted JSON shape of the `fill_value` field for `int64`: a JSON integer in [-2**63, 2**63 - 1].""" + + +# uint8: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +UINT8_DATA_TYPE_NAME: Final = "uint8" +"""The `data_type` value for the `uint8` type.""" + +Uint8DataTypeName = Literal["uint8"] +"""Literal type of the `data_type` field for `uint8`.""" + +Uint8FillValue = Annotated[int, Interval(ge=0, le=2**8 - 1)] +"""Permitted JSON shape of the `fill_value` field for `uint8`: a JSON integer in [0, 255].""" + + +# uint16: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +UINT16_DATA_TYPE_NAME: Final = "uint16" +"""The `data_type` value for the `uint16` type.""" + +Uint16DataTypeName = Literal["uint16"] +"""Literal type of the `data_type` field for `uint16`.""" + +Uint16FillValue = Annotated[int, Interval(ge=0, le=2**16 - 1)] +"""Permitted JSON shape of the `fill_value` field for `uint16`: a JSON integer in [0, 65535].""" + + +# uint32: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +UINT32_DATA_TYPE_NAME: Final = "uint32" +"""The `data_type` value for the `uint32` type.""" + +Uint32DataTypeName = Literal["uint32"] +"""Literal type of the `data_type` field for `uint32`.""" + +Uint32FillValue = Annotated[int, Interval(ge=0, le=2**32 - 1)] +"""Permitted JSON shape of the `fill_value` field for `uint32`: a JSON integer in [0, 2**32 - 1].""" + + +# uint64: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +UINT64_DATA_TYPE_NAME: Final = "uint64" +"""The `data_type` value for the `uint64` type.""" + +Uint64DataTypeName = Literal["uint64"] +"""Literal type of the `data_type` field for `uint64`.""" + +Uint64FillValue = Annotated[int, Interval(ge=0, le=2**64 - 1)] +"""Permitted JSON shape of the `fill_value` field for `uint64`: a JSON integer in [0, 2**64 - 1].""" + + +# float16: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +FLOAT16_DATA_TYPE_NAME: Final = "float16" +"""The `data_type` value for the `float16` type.""" + +Float16DataTypeName = Literal["float16"] +"""Literal type of the `data_type` field for `float16`.""" + +Float16SpecialFillValue = FloatSpecialFillValue +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" + +HexFloat16 = NewType("HexFloat16", str) +"""A 6-character hex string (`0x` + 4 hex digits) encoding the +unsigned-integer representation of a float16.""" + +Float16FillValue = float | int | Float16SpecialFillValue | HexFloat16 +"""Permitted JSON shape of the `fill_value` field for `float16`. + +Either a JSON number, one of the named non-finite sentinels (`"NaN"`, +`"Infinity"`, `"-Infinity"`), or a `HexFloat16` (`0xYYYY` string encoding +the unsigned-integer representation of the IEEE 754 value). +""" + +CANONICAL_NAN_HEX_FLOAT16: Final = "0x7e00" +"""Canonical hex form of the float16 NaN sentinel `"NaN"`. + +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the +most significant mantissa bit set, and all other mantissa bits zero +(the IEEE 754 default quiet NaN). Other NaN bit patterns must be +encoded with the explicit hex-string form. +""" + +CANONICAL_POSITIVE_INFINITY_HEX_FLOAT16: Final = "0x7c00" +"""Canonical hex form of the float16 `"Infinity"` sentinel.""" + +CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT16: Final = "0xfc00" +"""Canonical hex form of the float16 `"-Infinity"` sentinel.""" + + +# float32: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +FLOAT32_DATA_TYPE_NAME: Final = "float32" +"""The `data_type` value for the `float32` type.""" + +Float32DataTypeName = Literal["float32"] +"""Literal type of the `data_type` field for `float32`.""" + +Float32SpecialFillValue = FloatSpecialFillValue +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" + +HexFloat32 = NewType("HexFloat32", str) +"""A 10-character hex string (`0x` + 8 hex digits) encoding the +unsigned-integer representation of a float32.""" + +Float32FillValue = float | int | Float32SpecialFillValue | HexFloat32 +"""Permitted JSON shape of the `fill_value` field for `float32`. + +Either a JSON number, one of the named non-finite sentinels (`"NaN"`, +`"Infinity"`, `"-Infinity"`), or a `HexFloat32` (`0xYYYYYYYY` string +encoding the unsigned-integer representation of the IEEE 754 value). +""" + +CANONICAL_NAN_HEX_FLOAT32: Final = "0x7fc00000" +"""Canonical hex form of the float32 NaN sentinel `"NaN"`. + +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the +most significant mantissa bit set, and all other mantissa bits zero +(the IEEE 754 default quiet NaN). Other NaN bit patterns must be +encoded with the explicit hex-string form. +""" + +CANONICAL_POSITIVE_INFINITY_HEX_FLOAT32: Final = "0x7f800000" +"""Canonical hex form of the float32 `"Infinity"` sentinel.""" + +CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT32: Final = "0xff800000" +"""Canonical hex form of the float32 `"-Infinity"` sentinel.""" + + +# float64: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +FLOAT64_DATA_TYPE_NAME: Final = "float64" +"""The `data_type` value for the `float64` type.""" + +Float64DataTypeName = Literal["float64"] +"""Literal type of the `data_type` field for `float64`.""" + +Float64SpecialFillValue = FloatSpecialFillValue +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" + +HexFloat64 = NewType("HexFloat64", str) +"""An 18-character hex string (`0x` + 16 hex digits) encoding the +unsigned-integer representation of a float64.""" + +Float64FillValue = float | int | Float64SpecialFillValue | HexFloat64 +"""Permitted JSON shape of the `fill_value` field for `float64`. + +Either a JSON number, one of the named non-finite sentinels (`"NaN"`, +`"Infinity"`, `"-Infinity"`), or a `HexFloat64` (`0xYYYYYYYYYYYYYYYY` +string encoding the unsigned-integer representation of the IEEE 754 +value). +""" + +CANONICAL_NAN_HEX_FLOAT64: Final = "0x7ff8000000000000" +"""Canonical hex form of the float64 NaN sentinel `"NaN"`. + +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the +most significant mantissa bit set, and all other mantissa bits zero +(the IEEE 754 default quiet NaN). Other NaN bit patterns must be +encoded with the explicit hex-string form. +""" + +CANONICAL_POSITIVE_INFINITY_HEX_FLOAT64: Final = "0x7ff0000000000000" +"""Canonical hex form of the float64 `"Infinity"` sentinel.""" + +CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT64: Final = "0xfff0000000000000" +"""Canonical hex form of the float64 `"-Infinity"` sentinel.""" + + +# complex64: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +COMPLEX64_DATA_TYPE_NAME: Final = "complex64" +"""The `data_type` value for the `complex64` type.""" + +Complex64DataTypeName = Literal["complex64"] +"""Literal type of the `data_type` field for `complex64`.""" + +Complex64Component = Float32FillValue +"""One real or imaginary component of a `complex64` fill value. + +Same shape as a `float32` fill value: a JSON number, a named sentinel, +or a `HexFloat32` string. +""" + +Complex64FillValue = tuple[Complex64Component, Complex64Component] +"""Permitted JSON shape of the `fill_value` field for `complex64`. + +A two-element JSON array `[real, imag]` where each component is a +`Complex64Component`. +""" + + +# complex128: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +COMPLEX128_DATA_TYPE_NAME: Final = "complex128" +"""The `data_type` value for the `complex128` type.""" + +Complex128DataTypeName = Literal["complex128"] +"""Literal type of the `data_type` field for `complex128`.""" + +Complex128Component = Float64FillValue +"""One real or imaginary component of a `complex128` fill value. + +Same shape as a `float64` fill value: a JSON number, a named sentinel, +or a `HexFloat64` string. +""" + +Complex128FillValue = tuple[Complex128Component, Complex128Component] +"""Permitted JSON shape of the `fill_value` field for `complex128`. + +A two-element JSON array `[real, imag]` where each component is a +`Complex128Component`. +""" + + +# raw: https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html + +RawBytesDataTypeName = NewType("RawBytesDataTypeName", str) +"""A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`). + +"raw bits, variable size given by *, limited to be a multiple of 8": + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47 +""" + +RawBytesFillValue = tuple[ByteValue, ...] +"""Permitted JSON shape of the `fill_value` field for `r`. + +A JSON array of N/8 integers in `[0, 255]` (one per byte). +""" + + +class RawBytesConfiguration(TypedDict, closed=True): + """The configuration that a raw-bytes name carries: its size in bits, such as `{"bits": 16}` for `r16`. + + A document writes the name, never this object. The definition of `r*` + reads a name into this object, and the simplest spelling writes it + back as the name. + """ + + bits: int + """The size in bits that the name writes, `16` for `r16`. + + A positive multiple of 8. + """ + + +# bytes: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md + +BYTES_DATA_TYPE_NAME: Final = "bytes" +"""The `data_type` value for the variable-length `bytes` type.""" + +BytesDataTypeName = Literal["bytes"] +"""Literal type of the `data_type` field for `bytes`.""" + +BytesFillValue = tuple[ByteValue, ...] | Base64Bytes +"""Permitted JSON shape of the `fill_value` field for `bytes`. + +Either a JSON array of integers in `[0, 255]` (one per byte), or a +`Base64Bytes` string encoding the byte sequence. +""" + + +# string: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/string/README.md + +STRING_DATA_TYPE_NAME: Final = "string" +"""The `data_type` value for the `string` type.""" + +StringDataTypeName = Literal["string"] +"""Literal type of the `data_type` field for `string`.""" + +StringFillValue = str +"""Permitted JSON shape of the `fill_value` field for `string`: a JSON unicode string.""" + + +# variable_length_bytes: https://github.com/zarr-developers/zarr-python/blob/a2f2415c86f7d35212ae8c0d5dc5e4a22a6c6191/src/zarr/core/dtype/npy/bytes.py#L1141-L1143 + +VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME: Final = "variable_length_bytes" +"""The `data_type` value for the `variable_length_bytes` type.""" + +VariableLengthBytesDataTypeName = Literal["variable_length_bytes"] +"""Literal type of the `data_type` field for `variable_length_bytes`.""" + +VariableLengthBytesFillValue = Base64Bytes +"""Permitted JSON shape of the `fill_value` field for `variable_length_bytes`: a `Base64Bytes` string, which is what zarr-python writes and reads.""" + + +# fixed_length_utf32: https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/data-types/fixed_length_utf32/README.md + +FIXED_LENGTH_UTF32_DATA_TYPE_NAME: Final = "fixed_length_utf32" +"""The `name` field value of the `fixed_length_utf32` data type.""" + +FixedLengthUtf32DataTypeName = Literal["fixed_length_utf32"] +"""Literal type of the `name` field of the `fixed_length_utf32` data type.""" + + +class FixedLengthUtf32Configuration(TypedDict, closed=True): + """Configuration for the `fixed_length_utf32` data type.""" + + length_bytes: ReadOnly[int] + """The size in bytes of every value: a positive multiple of 4, since + each code point takes 4 bytes.""" + + +class FixedLengthUtf32(TypedDict, closed=True): + """`fixed_length_utf32` data type metadata.""" + + name: FixedLengthUtf32DataTypeName + """The data type's name, `"fixed_length_utf32"`.""" + configuration: FixedLengthUtf32Configuration + """The data type's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ + + +FixedLengthUtf32FillValue = str +"""Permitted JSON shape of the `fill_value` field for `fixed_length_utf32`: a JSON string of at most `length_bytes / 4` code points.""" + + +# null_terminated_bytes: https://github.com/zarr-developers/zarr-python/blob/a2f2415c86f7d35212ae8c0d5dc5e4a22a6c6191/src/zarr/core/dtype/npy/bytes.py#L385-L390 + +NULL_TERMINATED_BYTES_DATA_TYPE_NAME: Final = "null_terminated_bytes" +"""The `name` field value of the `null_terminated_bytes` data type.""" + +NullTerminatedBytesDataTypeName = Literal["null_terminated_bytes"] +"""Literal type of the `name` field of the `null_terminated_bytes` data type.""" + + +class NullTerminatedBytesConfiguration(TypedDict, closed=True): + """Configuration for the `null_terminated_bytes` data type.""" + + length_bytes: ReadOnly[int] + """The size in bytes of every value, at least 1.""" + + +class NullTerminatedBytes(TypedDict, closed=True): + """`null_terminated_bytes` data type metadata.""" + + name: NullTerminatedBytesDataTypeName + """The data type's name, `"null_terminated_bytes"`.""" + configuration: NullTerminatedBytesConfiguration + """The data type's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ + + +NullTerminatedBytesFillValue = Base64Bytes +"""Permitted JSON shape of the `fill_value` field for `null_terminated_bytes`: the bytes, without their trailing null bytes, as a `Base64Bytes` string.""" + + +# zarr_python_raw_bytes: https://github.com/zarr-developers/zarr-python/blob/a2f2415c86f7d35212ae8c0d5dc5e4a22a6c6191/src/zarr/core/dtype/npy/bytes.py#L783-L785 + +ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME: Final = "raw_bytes" +"""The `name` field value of zarr-python's `raw_bytes` data type.""" + +ZarrPythonRawBytesDataTypeName = Literal["raw_bytes"] +"""Literal type of the `name` field of zarr-python's `raw_bytes` data type.""" + + +class ZarrPythonRawBytesConfiguration(TypedDict, closed=True): + """Configuration for zarr-python's `raw_bytes` data type.""" + + length_bytes: ReadOnly[int] + """The size in bytes of every value, at least 1.""" + + +class ZarrPythonRawBytes(TypedDict, closed=True): + """zarr-python's `raw_bytes` data type metadata.""" + + name: ZarrPythonRawBytesDataTypeName + """The data type's name, `"raw_bytes"`.""" + configuration: ZarrPythonRawBytesConfiguration + """The data type's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ + + +ZarrPythonRawBytesFillValue = Base64Bytes +"""Permitted JSON shape of the `fill_value` field for zarr-python's `raw_bytes`: the bytes as a `Base64Bytes` string, which zarr-python writes `length_bytes` long.""" + + +# numpy_datetime64: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md + +NUMPY_DATETIME64_DATA_TYPE_NAME: Final = "numpy.datetime64" +"""The `name` field value of the `numpy.datetime64` data type.""" + +NumpyDatetime64DataTypeName = Literal["numpy.datetime64"] +"""Literal type of the `name` field of the `numpy.datetime64` data type.""" + + +class NumpyDatetime64Configuration(TypedDict, closed=True): + """Configuration for the `numpy.datetime64` data type.""" + + unit: ReadOnly[NumpyTimeUnit] + """A string encoding a unit of time. + + `"us"` and `"μs"` are equivalent spellings of microseconds; `"generic"` + means the unit is unspecified. + """ + scale_factor: ReadOnly[NumpyTimeScaleFactor] + """The multiplier relative to the unit. + + An integer in `[1, 2147483647]`. + """ + + +class NumpyDatetime64(TypedDict, closed=True): + """`numpy.datetime64` data type metadata.""" + + name: NumpyDatetime64DataTypeName + """The data type's name, `"numpy.datetime64"`.""" + configuration: NumpyDatetime64Configuration + """The data type's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ + + +NumpyDatetime64FillValue = NumpyTimeTicks | Literal["NaT"] +"""Permitted JSON shape of the `fill_value` field for `numpy.datetime64`. + +Either a JSON integer (count of `unit * scale_factor` since the epoch), +or the string `"NaT"` (equivalent to the integer `-2**63`). +""" + + +# numpy_timedelta64: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.timedelta64/README.md + +NUMPY_TIMEDELTA64_DATA_TYPE_NAME: Final = "numpy.timedelta64" +"""The `name` field value of the `numpy.timedelta64` data type.""" + +NumpyTimedelta64DataTypeName = Literal["numpy.timedelta64"] +"""Literal type of the `name` field of the `numpy.timedelta64` data type.""" + + +class NumpyTimedelta64Configuration(TypedDict, closed=True): + """Configuration for the `numpy.timedelta64` data type.""" + + unit: ReadOnly[NumpyTimeUnit] + """A string encoding a unit of time. + + `"us"` and `"μs"` are equivalent spellings of microseconds; `"generic"` + means the unit is unspecified. + """ + scale_factor: ReadOnly[NumpyTimeScaleFactor] + """The multiplier relative to the unit. + + An integer in `[1, 2147483647]`. + """ + + +class NumpyTimedelta64(TypedDict, closed=True): + """`numpy.timedelta64` data type metadata.""" + + name: NumpyTimedelta64DataTypeName + """The data type's name, `"numpy.timedelta64"`.""" + configuration: NumpyTimedelta64Configuration + """The data type's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ + + +NumpyTimedelta64FillValue = NumpyTimeTicks | Literal["NaT"] +"""Permitted JSON shape of the `fill_value` field for `numpy.timedelta64`. + +Either a JSON integer (a count of `unit * scale_factor`), or the string +`"NaT"` (equivalent to the integer `-2**63`). +""" + + +# struct: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md + +STRUCT_DATA_TYPE_NAME: Final = "struct" +"""The `name` field value of the `struct` data type.""" + +StructDataTypeName = Literal["struct"] +"""Literal type of the `name` field of the `struct` data type.""" + + +class StructField(TypedDict, closed=True): + """A single field entry inside a structured dtype.""" + + name: ReadOnly[str] + """The field name (must be unique within a struct and non-empty).""" + data_type: ReadOnly[ZarrV3DataTypeDeclaration] + """The field's data type. + + Recursive: may be a bare-string primitive or a named-config envelope + including another `struct`. + """ + + +class StructConfiguration(TypedDict, closed=True): + """Configuration for the `struct` data type.""" + + fields: ReadOnly[tuple[StructField, ...]] + """The record's fields, in the order they are encoded. + + At least one. The order is significant: it is the order in which the + `bytes` codec packs the fields' values. + """ + + +class Struct(TypedDict, closed=True): + """`struct` data type metadata.""" + + name: StructDataTypeName + """The data type's name, `"struct"`.""" + configuration: StructConfiguration + """The data type's parameters.""" + must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ + + +StructFillValue = Mapping[str, JSONValue] +"""Permitted JSON shape of the `fill_value` field for `struct`. + +A JSON object mapping each field name to that field's fill value. Field +fill values are themselves shaped per the field's `data_type`, recursively. +""" + + +__all__ = [ + "BOOL_DATA_TYPE_NAME", + "BYTES_DATA_TYPE_NAME", + "CANONICAL_NAN_HEX_FLOAT16", + "CANONICAL_NAN_HEX_FLOAT32", + "CANONICAL_NAN_HEX_FLOAT64", + "CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT16", + "CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT32", + "CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT64", + "CANONICAL_POSITIVE_INFINITY_HEX_FLOAT16", + "CANONICAL_POSITIVE_INFINITY_HEX_FLOAT32", + "CANONICAL_POSITIVE_INFINITY_HEX_FLOAT64", + "COMPLEX64_DATA_TYPE_NAME", + "COMPLEX128_DATA_TYPE_NAME", + "FIXED_LENGTH_UTF32_DATA_TYPE_NAME", + "FLOAT16_DATA_TYPE_NAME", + "FLOAT32_DATA_TYPE_NAME", + "FLOAT64_DATA_TYPE_NAME", + "FLOAT_SPECIAL_FILL_VALUES", + "INT8_DATA_TYPE_NAME", + "INT16_DATA_TYPE_NAME", + "INT32_DATA_TYPE_NAME", + "INT64_DATA_TYPE_NAME", + "NULL_TERMINATED_BYTES_DATA_TYPE_NAME", + "NUMPY_DATETIME64_DATA_TYPE_NAME", + "NUMPY_TIMEDELTA64_DATA_TYPE_NAME", + "NUMPY_TIME_UNIT", + "STRING_DATA_TYPE_NAME", + "STRUCT_DATA_TYPE_NAME", + "UINT8_DATA_TYPE_NAME", + "UINT16_DATA_TYPE_NAME", + "UINT32_DATA_TYPE_NAME", + "UINT64_DATA_TYPE_NAME", + "VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME", + "ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME", + "BoolDataTypeName", + "BoolFillValue", + "ByteValue", + "BytesDataTypeName", + "BytesFillValue", + "Complex64Component", + "Complex64DataTypeName", + "Complex64FillValue", + "Complex128Component", + "Complex128DataTypeName", + "Complex128FillValue", + "FixedLengthUtf32", + "FixedLengthUtf32Configuration", + "FixedLengthUtf32DataTypeName", + "FixedLengthUtf32FillValue", + "Float16DataTypeName", + "Float16FillValue", + "Float16SpecialFillValue", + "Float32DataTypeName", + "Float32FillValue", + "Float32SpecialFillValue", + "Float64DataTypeName", + "Float64FillValue", + "Float64SpecialFillValue", + "FloatSpecialFillValue", + "HexFloat16", + "HexFloat32", + "HexFloat64", + "Int8DataTypeName", + "Int8FillValue", + "Int16DataTypeName", + "Int16FillValue", + "Int32DataTypeName", + "Int32FillValue", + "Int64DataTypeName", + "Int64FillValue", + "NullTerminatedBytes", + "NullTerminatedBytesConfiguration", + "NullTerminatedBytesDataTypeName", + "NullTerminatedBytesFillValue", + "NumpyDatetime64", + "NumpyDatetime64Configuration", + "NumpyDatetime64DataTypeName", + "NumpyDatetime64FillValue", + "NumpyTimeUnit", + "NumpyTimedelta64", + "NumpyTimedelta64Configuration", + "NumpyTimedelta64DataTypeName", + "NumpyTimedelta64FillValue", + "RawBytesConfiguration", + "RawBytesDataTypeName", + "RawBytesFillValue", + "StringDataTypeName", + "StringFillValue", + "Struct", + "StructConfiguration", + "StructDataTypeName", + "StructField", + "StructFillValue", + "Uint8DataTypeName", + "Uint8FillValue", + "Uint16DataTypeName", + "Uint16FillValue", + "Uint32DataTypeName", + "Uint32FillValue", + "Uint64DataTypeName", + "Uint64FillValue", + "VariableLengthBytesDataTypeName", + "VariableLengthBytesFillValue", + "ZarrPythonRawBytes", + "ZarrPythonRawBytesConfiguration", + "ZarrPythonRawBytesDataTypeName", + "ZarrPythonRawBytesFillValue", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index a893cae7bf..0b6db11ac1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -5,10 +5,7 @@ """ from collections.abc import Iterator -from typing import Annotated, Final, Literal, NotRequired - -from annotated_types import Ge -from typing_extensions import TypedDict +from typing import Final from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested @@ -16,62 +13,14 @@ ChunkGridDefinition, Lengths, ) - -RECTILINEAR_CHUNK_GRID_NAME: Final = "rectilinear" -"""The `name` field value of the rectilinear chunk grid.""" - -RectilinearChunkGridName = Literal["rectilinear"] -"""Literal type of the `name` field of the rectilinear chunk grid.""" - -_Positive = Annotated[int, Ge(1)] - -RectilinearDimSpec = _Positive | tuple[_Positive | tuple[_Positive, _Positive], ...] -"""JSON shape for one dimension's rectilinear spec. - -Either a bare integer (uniform shorthand for a regular dimension within -a rectilinear grid), or a tuple of integers and/or `[value, count]` RLE -pairs. Every extent, and every run's length and count, is at least 1. -""" - - -class RectilinearChunkGridConfiguration(TypedDict, closed=True): - """Configuration for the rectilinear chunk grid.""" - - kind: Literal["inline"] - """How `chunk_shapes` is encoded; `"inline"` is the only kind the spec defines.""" - chunk_shapes: tuple[RectilinearDimSpec, ...] - """The chunk edge lengths along each dimension of the array, one entry per dimension. - - An entry is a bare integer, a step repeated until it covers the - dimension, or a list of edge lengths and `[length, count]` runs, whose - lengths must sum to at least the dimension's length. Every length and - count is at least 1. - """ - - -class RectilinearChunkGridObject(TypedDict, closed=True): - """Rectilinear chunk grid metadata in object form.""" - - name: RectilinearChunkGridName - """The chunk grid's name, `"rectilinear"`.""" - configuration: RectilinearChunkGridConfiguration - """The chunk grid's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this chunk grid must refuse the document. - - Absent means `true`. The spec does not permit `false` for a chunk grid - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). - """ - - -RectilinearChunkGridMetadata = RectilinearChunkGridObject -"""Permitted JSON shape for rectilinear chunk grid metadata. - -`kind` and `chunk_shapes` are required, so only the object form is valid; -the short-hand-name form is not permitted by the spec for this grid. - https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md#L59-L62 - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 -""" +from zarr_metadata.v3._types.chunk_grid import ( + RECTILINEAR_CHUNK_GRID_NAME, + RectilinearChunkGridConfiguration, + RectilinearChunkGridMetadata, + RectilinearChunkGridName, + RectilinearChunkGridObject, + RectilinearDimSpec, +) def canonical_dim_spec(spec: RectilinearDimSpec) -> RectilinearDimSpec: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 365e27ac74..88b65d1226 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -5,10 +5,7 @@ """ from collections.abc import Iterator -from typing import Annotated, Final, Literal, NotRequired - -from annotated_types import Ge -from typing_extensions import TypedDict +from typing import Final from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested @@ -16,57 +13,13 @@ ChunkGridDefinition, Lengths, ) - -REGULAR_CHUNK_GRID_NAME: Final = "regular" -"""The `name` field value of the regular chunk grid.""" - -RegularChunkGridName = Literal["regular"] -"""Literal type of the `name` field of the regular chunk grid.""" - - -class RegularChunkGridConfiguration(TypedDict, closed=True): - """Configuration for the regular chunk grid. - - Every chunk length is at least 1, even along a dimension of length 0: - "Chunk sizes must be greater than zero" - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L40). - The core spec's "The chunk shape elements are non-zero when the - corresponding dimensions of the arrays have non-zero length" - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L284-L285) - says less about an empty dimension, and it does not allow anything - that the grid spec forbids. - """ - - chunk_shape: tuple[Annotated[int, Ge(1)], ...] - """The length of every chunk along each dimension of the array. - - One entry per dimension, each at least 1. - """ - - -class RegularChunkGridObject(TypedDict, closed=True): - """Regular chunk grid metadata in object form.""" - - name: RegularChunkGridName - """The chunk grid's name, `"regular"`.""" - configuration: RegularChunkGridConfiguration - """The chunk grid's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this chunk grid must refuse the document. - - Absent means `true`. The spec does not permit `false` for a chunk grid - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). - """ - - -RegularChunkGridMetadata = RegularChunkGridObject -"""Permitted JSON shape for regular chunk grid metadata. - -`chunk_shape` is required and has no default, so only the object form is -valid; the short-hand-name form is not permitted by the spec for this grid. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L528-L537 ("must be an object with the names name and configuration") - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 -""" +from zarr_metadata.v3._types.chunk_grid import ( + REGULAR_CHUNK_GRID_NAME, + RegularChunkGridConfiguration, + RegularChunkGridMetadata, + RegularChunkGridName, + RegularChunkGridObject, +) def _shape_rules( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 1e95a07a19..693c4c4720 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -7,66 +7,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding """ -from typing import Final, Literal, NotRequired - -from typing_extensions import TypedDict +from typing import Final from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition - -DEFAULT_CHUNK_KEY_ENCODING_NAME: Final = "default" -"""The `name` field value of the default chunk key encoding.""" - -DefaultChunkKeyEncodingName = Literal["default"] -"""Literal type of the `name` field of the default chunk key encoding.""" - -DefaultChunkKeyEncodingSeparator = Literal["/", "."] -"""Literal type of permitted `separator` values for the default chunk key encoding. - -Defaults to `"/"` if absent. -""" - -DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR: Final = ("/", ".") -"""Tuple of permitted values for the `separator` field of the default chunk key encoding.""" - - -class DefaultChunkKeyEncodingConfiguration(TypedDict, closed=True): - """Configuration for the default chunk key encoding. - - `separator` is optional and defaults to `"/"` per spec. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst#L27-L29 - """ - - separator: NotRequired[DefaultChunkKeyEncodingSeparator] - """The character between the parts of a chunk key: `c/1/23/45` or `c.1.23.45`. - - Absent means `"/"`. - """ - - -class DefaultChunkKeyEncodingObject(TypedDict, closed=True): - """Default chunk key encoding metadata in object form.""" - - name: DefaultChunkKeyEncodingName - """The chunk key encoding's name, `"default"`.""" - configuration: NotRequired[DefaultChunkKeyEncodingConfiguration] - """The chunk key encoding's parameters. - - Absent means every parameter takes its default: a `separator` of `"/"`. - """ - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this chunk key encoding must refuse the document. - - Absent means `true`. The spec does not permit `false` for a chunk key encoding - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). - """ - - -DefaultChunkKeyEncodingMetadata = DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName -"""Permitted JSON shapes for the default chunk-key encoding metadata. - -The configuration has no required keys (`separator` defaults to `"/"`), -so the short-hand-name form is permitted in addition to the object form. -""" +from zarr_metadata.v3._types.chunk_key_encoding import ( + DEFAULT_CHUNK_KEY_ENCODING_NAME, + DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, + DefaultChunkKeyEncodingConfiguration, + DefaultChunkKeyEncodingMetadata, + DefaultChunkKeyEncodingName, + DefaultChunkKeyEncodingObject, + DefaultChunkKeyEncodingSeparator, +) def _canonical( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 1162699a6b..24c458d102 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -13,66 +13,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding """ -from typing import Final, Literal, NotRequired - -from typing_extensions import TypedDict +from typing import Final from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition - -V2_CHUNK_KEY_ENCODING_NAME: Final = "v2" -"""The `name` field value of the v2 chunk key encoding.""" - -V2ChunkKeyEncodingName = Literal["v2"] -"""Literal type of the `name` field of the v2 chunk key encoding.""" - -V2ChunkKeyEncodingSeparator = Literal["/", "."] -"""Literal type of permitted `separator` values for the v2 chunk key encoding. - -Defaults to `"."` if absent. -""" - -V2_CHUNK_KEY_ENCODING_SEPARATOR: Final = ("/", ".") -"""Tuple of permitted values for the `separator` field of the v2 chunk key encoding.""" - - -class V2ChunkKeyEncodingConfiguration(TypedDict, closed=True): - """Configuration for the v2 chunk key encoding. - - `separator` is optional and defaults to `"."` per spec. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst#L27-L29 - """ - - separator: NotRequired[V2ChunkKeyEncodingSeparator] - """The character between the parts of a chunk key: `1.23.45` or `1/23/45`. - - Absent means `"."`. - """ - - -class V2ChunkKeyEncodingObject(TypedDict, closed=True): - """v2-compatibility chunk key encoding metadata in object form.""" - - name: V2ChunkKeyEncodingName - """The chunk key encoding's name, `"v2"`.""" - configuration: NotRequired[V2ChunkKeyEncodingConfiguration] - """The chunk key encoding's parameters. - - Absent means every parameter takes its default: a `separator` of `"."`. - """ - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this chunk key encoding must refuse the document. - - Absent means `true`. The spec does not permit `false` for a chunk key encoding - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). - """ - - -V2ChunkKeyEncodingMetadata = V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName -"""Permitted JSON shapes for the v2-compatibility chunk-key encoding metadata. - -The configuration has no required keys (`separator` defaults to `"."`), -so the short-hand-name form is permitted in addition to the object form. -""" +from zarr_metadata.v3._types.chunk_key_encoding import ( + V2_CHUNK_KEY_ENCODING_NAME, + V2_CHUNK_KEY_ENCODING_SEPARATOR, + V2ChunkKeyEncodingConfiguration, + V2ChunkKeyEncodingMetadata, + V2ChunkKeyEncodingName, + V2ChunkKeyEncodingObject, + V2ChunkKeyEncodingSeparator, +) def _canonical(configuration: V2ChunkKeyEncodingConfiguration) -> V2ChunkKeyEncodingConfiguration: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py index cd31374272..f56da6052e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py @@ -21,16 +21,18 @@ from typing import Final from zarr_metadata.v3._definition import RAW_BYTES_NAME -from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.fixed_length_utf32 import FIXED_LENGTH_UTF32_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.null_terminated_bytes import NULL_TERMINATED_BYTES_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.variable_length_bytes import VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.zarr_python_raw_bytes import ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME +from zarr_metadata.v3._types.data_type import ( + BOOL_DATA_TYPE_NAME, + BYTES_DATA_TYPE_NAME, + COMPLEX64_DATA_TYPE_NAME, + COMPLEX128_DATA_TYPE_NAME, + FIXED_LENGTH_UTF32_DATA_TYPE_NAME, + NULL_TERMINATED_BYTES_DATA_TYPE_NAME, + STRING_DATA_TYPE_NAME, + STRUCT_DATA_TYPE_NAME, + VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME, + ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME, +) FLOATING_POINT: Final = frozenset( { diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index f5672e9e61..5aac8b054d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -5,97 +5,23 @@ """ from collections.abc import Iterator -from typing import Annotated, Final, Literal, NotRequired - -from annotated_types import Ge, Interval -from typing_extensions import TypedDict +from typing import Final from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import CodecDefinition - -BLOSC_CODEC_NAME: Final = "blosc" -"""The `name` field value of the `blosc` codec.""" - -BloscCodecName = Literal["blosc"] -"""Literal type of the `name` field of the `blosc` codec.""" - -BloscShuffle = Literal["noshuffle", "shuffle", "bitshuffle"] -"""Literal type of blosc shuffle mode names.""" - -BLOSC_SHUFFLE: Final = ("noshuffle", "shuffle", "bitshuffle") -"""Tuple of permitted values for the `shuffle` field of the `blosc` codec.""" - -BloscCName = Literal["lz4", "lz4hc", "blosclz", "snappy", "zlib", "zstd"] -"""Literal type of blosc compressor identifiers.""" - -BLOSC_CNAME: Final = ("lz4", "lz4hc", "blosclz", "snappy", "zlib", "zstd") -"""Tuple of permitted values for the `cname` field of the `blosc` codec.""" - - -class BloscCodecConfiguration(TypedDict, closed=True): - """Configuration for the Zarr v3 `blosc` codec.""" - - cname: BloscCName - """The compressor blosc applies to each block, one of `BLOSC_CNAME`.""" - clevel: Annotated[int, Interval(ge=0, le=9)] - """The compression level, an integer from 0 to 9. - - 0 turns compression off; 1 is the fastest and compresses least, 9 the - slowest and compresses most. - """ - shuffle: BloscShuffle - """The shuffling applied before compression. - - `"noshuffle"` for none, `"shuffle"` for byte-wise, `"bitshuffle"` for - bit-wise. - """ - blocksize: Annotated[int, Ge(0)] - """The size in bytes of the blocks the buffer is divided into before compression. - - At least 0; 0 lets blosc choose the size. - """ - typesize: NotRequired[int] - """The stride in bytes over which shuffling is performed. - - Required, and at least 1, unless `shuffle` is `"noshuffle"`, under - which the spec ignores it and `canonicalize` drops it. - """ - - -class BloscCodecObject(TypedDict, closed=True): - """`blosc` codec metadata in object form.""" - - name: BloscCodecName - """The codec's name, `"blosc"`.""" - configuration: BloscCodecConfiguration - """The codec's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -BloscCodecMetadata = BloscCodecObject -"""Permitted JSON shape for `blosc` codec metadata. - -The configuration has multiple required keys (`cname`, `clevel`, `shuffle`, -`blocksize`), so only the object form is valid; the short-hand-name form -is not permitted by the spec for this codec. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/blosc/index.rst#L57-L98 (configuration parameters) - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") -""" - -BLOSC_NO_SHUFFLE: Final = "noshuffle" -"""The `shuffle` value under which `typesize` carries no information. - -The spec requires `typesize` "unless `shuffle` is `"noshuffle"`, in which -case the value is ignored", so this is the one value that changes whether -another member is required. -""" +from zarr_metadata.v3._types.codec import ( + BLOSC_CNAME, + BLOSC_CODEC_NAME, + BLOSC_NO_SHUFFLE, + BLOSC_SHUFFLE, + BloscCName, + BloscCodecConfiguration, + BloscCodecMetadata, + BloscCodecName, + BloscCodecObject, + BloscShuffle, +) def _rules(configuration: BloscCodecConfiguration, nested: Nested) -> Iterator[ValidationProblem]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 037d702ad1..652c3d9f5f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -5,9 +5,7 @@ """ from collections.abc import Iterator -from typing import Final, Literal, NotRequired - -from typing_extensions import TypedDict +from typing import Final from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import Nested @@ -16,71 +14,15 @@ CodecDefinition, storage_of, ) - -BYTES_CODEC_NAME: Final = "bytes" -"""The `name` field value of the `bytes` codec.""" - -BytesCodecName = Literal["bytes"] -"""Literal type of the `name` field of the `bytes` codec.""" - -Endianness = Literal["little", "big"] -"""Literal type of byte order of multi-byte numeric data.""" - -ENDIANNESS: Final = ("little", "big") -"""Tuple of permitted values for the `endian` field of the `bytes` codec.""" - - -class BytesCodecConfiguration(TypedDict, closed=True): - """ - Configuration for the Zarr v3 `bytes` codec. - - The `endian` field is required for multi-byte data types. - """ - - endian: NotRequired[Endianness] - """The byte order in which each value is written. - - Required when the values of the data type the codec is handed take more - than one byte; a single-byte type such as `uint8` or `bool` does not - depend on it. - """ - - -class BytesCodecObject(TypedDict, closed=True): - """`bytes` codec metadata in object form. - - `configuration` is itself optional — when no configuration fields are - set, the entire `configuration` key may be omitted. This matches the - bare-string short-hand form (`BytesCodecName`) at the canonical data - level; both encodings describe a `bytes` codec with default settings. - """ - - name: BytesCodecName - """The codec's name, `"bytes"`.""" - configuration: NotRequired[BytesCodecConfiguration] - """The codec's parameters. - - May be absent when no parameter is set. - """ - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -BytesCodecMetadata = BytesCodecObject | BytesCodecName -"""Permitted JSON shapes for `bytes` codec metadata. - -The configuration has no required keys (`endian` is conditionally required -at runtime based on data type), so the spec's short-hand-name form is -permitted in addition to the object form, and the object form may itself -omit `configuration` entirely. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69 ("endian: Required for data types for which endianness is applicable") - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 -""" +from zarr_metadata.v3._types.codec import ( + BYTES_CODEC_NAME, + ENDIANNESS, + BytesCodecConfiguration, + BytesCodecMetadata, + BytesCodecName, + BytesCodecObject, + Endianness, +) def _chunk_rules( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 114d31fddf..c00af8b8b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -5,9 +5,7 @@ """ from collections.abc import Iterator -from typing import Final, Literal, NotRequired - -from typing_extensions import TypedDict +from typing import Final, Literal from zarr_metadata._common import JSONValue from zarr_metadata._json import ValidationProblem, shown @@ -16,133 +14,24 @@ Nested, fill_value_problems, ) -from zarr_metadata.v3._common import ZarrV3DataTypeDeclaration from zarr_metadata.v3._definition import ( Chunk, CodecDefinition, ) -from zarr_metadata.v3.codec._arithmetic import FLOATING_POINT, NO_REAL_NUMBERS - -CAST_VALUE_CODEC_NAME: Final = "cast_value" -"""The `name` field value of the `cast_value` codec.""" - -CastValueCodecName = Literal["cast_value"] -"""Literal type of the `name` field of the `cast_value` codec.""" - -CastRoundingMode = Literal[ - "nearest-even", - "towards-zero", - "towards-positive", - "towards-negative", - "nearest-away", -] -"""Literal type of permitted values for the `rounding` configuration field. - -Defaults to `"nearest-even"` if absent. -""" - -CAST_ROUNDING_MODE: Final = ( - "nearest-even", - "towards-zero", - "towards-positive", - "towards-negative", - "nearest-away", +from zarr_metadata.v3._types.codec import ( + CAST_OUT_OF_RANGE_MODE, + CAST_ROUNDING_MODE, + CAST_VALUE_CODEC_NAME, + CastOutOfRangeMode, + CastRoundingMode, + CastValueCodecConfiguration, + CastValueCodecMetadata, + CastValueCodecName, + CastValueCodecObject, + ScalarMap, + ScalarMapEntry, ) -"""Tuple of permitted values for the `rounding` field of the `cast_value` codec.""" - -CastOutOfRangeMode = Literal["clamp", "wrap"] -"""Literal type of permitted values for the `out_of_range` configuration field. - -If absent, out-of-range values are an encoding/decoding error. -""" - -CAST_OUT_OF_RANGE_MODE: Final = ("clamp", "wrap") -"""Tuple of permitted values for the `out_of_range` field of the `cast_value` codec.""" - -ScalarMapEntry = tuple[JSONValue, JSONValue] -"""A single `[input, output]` mapping in a `scalar_map` direction. - -Each scalar is encoded as a fill value of its data type. For a float, -the infinities are `"Infinity"` and `"-Infinity"`, as the core encoding -writes them. The codec's own example writes `"+Infinity"` instead. -""" - - -class ScalarMap(TypedDict, closed=True): - """Optional encode/decode scalar overrides for the cast_value codec.""" - - encode: NotRequired[tuple[ScalarMapEntry, ...]] - """The `[input, output]` pairs applied when encoding: input in the array's data type, output in `data_type`. - - Each scalar is written in its data type's fill value encoding. Where an - input repeats, the first pair applies. - """ - decode: NotRequired[tuple[ScalarMapEntry, ...]] - """The `[input, output]` pairs applied when decoding: input in `data_type`, output in the array's data type. - - Each scalar is written in its data type's fill value encoding. Where an - input repeats, the first pair applies. - """ - - -class CastValueCodecConfiguration(TypedDict, closed=True): - """ - Configuration for the Zarr v3 `cast_value` codec. - - `data_type` is the target data type that input values are cast to. It - is the same shape as the top-level array `data_type` field: either a - bare-string primitive name or a `{name, configuration}` envelope. - """ - - data_type: ZarrV3DataTypeDeclaration - """The data type values are cast to when encoding, and the one the decoder is handed. - - The codec is defined only for data types that model real numbers, the - integer and floating-point types: a boolean, raw-bits, string, bytes, - struct or complex type is refused, here and as the data type the codec - is handed. - """ - rounding: NotRequired[CastRoundingMode] - """How a value `data_type` cannot represent exactly is rounded. - - Absent means `"nearest-even"`. - """ - out_of_range: NotRequired[CastOutOfRangeMode] - """How a value outside the range of `data_type` is brought into it, after rounding. - - `"clamp"` takes the nearest bound; `"wrap"` takes the value congruent - modulo 2^N, N being the width in bits of `data_type`, and is permitted - only when `data_type` is an integral type. Absent means an out-of-range - value is an encoding or decoding error. - """ - scalar_map: NotRequired[ScalarMap] - """Scalars mapped explicitly, in place of `rounding` and `out_of_range`.""" - - -class CastValueCodecObject(TypedDict, closed=True): - """`cast_value` codec metadata in object form.""" - - name: CastValueCodecName - """The codec's name, `"cast_value"`.""" - configuration: CastValueCodecConfiguration - """The codec's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -CastValueCodecMetadata = CastValueCodecObject -"""Permitted JSON shape for `cast_value` codec metadata. - -`configuration.data_type` is required, so only the object form is valid; -the short-hand-name form is not permitted by the spec for this codec. - https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md#L33-L36 and #L46-L48 (required fields) - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") -""" +from zarr_metadata.v3.codec._arithmetic import FLOATING_POINT, NO_REAL_NUMBERS def _scalars( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 044f9fe1b8..0c94867d5a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -7,53 +7,16 @@ key is absent from the metadata. """ -from typing import Final, Literal, NotRequired +from typing import Final -from typing_extensions import TypedDict - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import CodecDefinition - -CRC32C_CODEC_NAME: Final = "crc32c" -"""The `name` field value of the `crc32c` codec.""" - -Crc32cCodecName = Literal["crc32c"] -"""Literal type of the `name` field of the `crc32c` codec.""" - - -class Crc32cCodecObject(TypedDict, closed=True): - """`crc32c` codec metadata in object form. - - Per spec the codec has no configuration fields. `configuration` is - optional and, if present, should be an empty mapping. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/crc32c/index.rst#L63-L66 - """ - - name: Crc32cCodecName - """The codec's name, `"crc32c"`.""" - configuration: NotRequired[EmptyConfiguration] - """The codec's parameters, of which there are none. - - Absent, or an empty mapping. - """ - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -Crc32cCodecMetadata = Crc32cCodecObject | Crc32cCodecName -"""Permitted JSON shapes for `crc32c` codec metadata. - -The spec's Extension definition allows extensions with no required -configuration to be encoded as a bare short-hand name. CRC32C has no -configuration, so both forms are valid. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 -""" - +from zarr_metadata.v3._types.codec import ( + CRC32C_CODEC_NAME, + Crc32cCodecMetadata, + Crc32cCodecName, + Crc32cCodecObject, +) CRC32C_CODEC: Final = CodecDefinition( name=CRC32C_CODEC_NAME, configuration=EmptyConfiguration, kind="bytes_bytes", size="static" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 1dc7220cfb..126e778797 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -4,66 +4,16 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/gzip/index.html """ -from typing import Annotated, Final, Literal, NotRequired - -from annotated_types import Interval -from typing_extensions import TypedDict +from typing import Final from zarr_metadata.v3._definition import CodecDefinition - -GZIP_CODEC_NAME: Final = "gzip" -"""The `name` field value of the `gzip` codec.""" - -GzipCodecName = Literal["gzip"] -"""Literal type of the `name` field of the `gzip` codec.""" - - -class GzipCodecConfiguration(TypedDict, closed=True): - """ - Configuration for the Zarr v3 `gzip` codec. - - `level` is an integer in the range 0-9; 0 disables compression and 9 - is slowest with the best compression ratio. The codec's compressed - output depends on `level`, so metadata that omits it cannot - reproducibly identify the chunk bytes produced by a writer — `level` - is required for the metadata to fulfill its reproducibility role, - even though the spec text does not mark it required with RFC 2119 - keywords. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/gzip/index.rst#L57-L66 - """ - - level: Annotated[int, Interval(ge=0, le=9)] - """The compression level, an integer from 0 to 9. - - 0 turns compression off; 1 is the fastest and compresses least, 9 the - slowest and compresses most. - """ - - -class GzipCodecObject(TypedDict, closed=True): - """`gzip` codec metadata in object form.""" - - name: GzipCodecName - """The codec's name, `"gzip"`.""" - configuration: GzipCodecConfiguration - """The codec's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -GzipCodecMetadata = GzipCodecObject -"""Permitted JSON shape for `gzip` codec metadata. - -`configuration.level` is required (it determines the codec's output bytes -and is therefore part of the metadata's reproducibility contract), so -only the object form is valid; the short-hand-name form is not permitted. -""" - +from zarr_metadata.v3._types.codec import ( + GZIP_CODEC_NAME, + GzipCodecConfiguration, + GzipCodecMetadata, + GzipCodecName, + GzipCodecObject, +) GZIP_CODEC: Final = CodecDefinition( name=GZIP_CODEC_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 3d2146879a..473eeb4c1f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -5,11 +5,8 @@ """ from collections.abc import Iterator -from typing import Final, Literal, NotRequired +from typing import Final -from typing_extensions import TypedDict - -from zarr_metadata._common import JSONValue from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import ( AcceptedDeclaration, @@ -20,75 +17,15 @@ Chunk, CodecDefinition, ) +from zarr_metadata.v3._types.codec import ( + SCALE_OFFSET_CODEC_NAME, + ScaleOffsetCodecConfiguration, + ScaleOffsetCodecMetadata, + ScaleOffsetCodecName, + ScaleOffsetCodecObject, +) from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS -SCALE_OFFSET_CODEC_NAME: Final = "scale_offset" -"""The `name` field value of the `scale_offset` codec.""" - -ScaleOffsetCodecName = Literal["scale_offset"] -"""Literal type of the `name` field of the `scale_offset` codec.""" - - -class ScaleOffsetCodecConfiguration(TypedDict, closed=True): - """ - Configuration for the Zarr v3 `scale_offset` codec. - - Both fields are optional. A missing `offset` is the additive identity - (e.g. 0 for numeric types); a missing `scale` is the multiplicative - identity (e.g. 1). Each scalar is encoded as a fill value of the input - data type of the codec: an integer for an integer type, and for a - float type a number, `"NaN"`, `"Infinity"`, `"-Infinity"` or a hex - string. - """ - - offset: NotRequired[JSONValue] - """The quantity subtracted from each value when encoding. - - Absent means the additive identity, e.g. 0. A fill value of the data - type the codec is handed. - """ - scale: NotRequired[JSONValue] - """The quantity each value is multiplied by when encoding, after `offset` is subtracted. - - Absent means the multiplicative identity, e.g. 1. A fill value of the - data type the codec is handed. - """ - - -class ScaleOffsetCodecObject(TypedDict, closed=True): - """`scale_offset` codec metadata in object form. - - `configuration` is itself optional per spec — when both `offset` and - `scale` are at their identity defaults, the codec is a no-op and the - entire `configuration` field may be omitted. - https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md#L18 and #L35 - """ - - name: ScaleOffsetCodecName - """The codec's name, `"scale_offset"`.""" - configuration: NotRequired[ScaleOffsetCodecConfiguration] - """The codec's parameters. - - May be absent when neither `offset` nor `scale` is set, which makes the - codec a no-op. - """ - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -ScaleOffsetCodecMetadata = ScaleOffsetCodecObject | ScaleOffsetCodecName -"""Permitted JSON shapes for `scale_offset` codec metadata. - -The configuration has no required keys (both `offset` and `scale` are -optional, and the configuration itself is optional), so the short-hand-name -form is permitted in addition to the object form. -""" - def _rules( configuration: ScaleOffsetCodecConfiguration, nested: Nested diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index ffd7239e78..5b4b3556e8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -5,111 +5,29 @@ """ from collections.abc import Iterator, Mapping -from typing import Annotated, Final, Literal, NotRequired - -from annotated_types import Ge -from typing_extensions import TypedDict +from typing import Final from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import ( AcceptedDeclaration, Nested, ) -from zarr_metadata.v3._common import ( - ZarrV3CodecDeclaration, - ZarrV3StaticCodecDeclaration, -) from zarr_metadata.v3._definition import ( Chunk, CodecDefinition, Lengths, ) -from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE, UINT64_DATA_TYPE_NAME - -SHARDING_INDEXED_CODEC_NAME: Final = "sharding_indexed" -"""The `name` field value of the `sharding_indexed` codec.""" - -ShardingIndexedCodecName = Literal["sharding_indexed"] -"""Literal type of the `name` field of the `sharding_indexed` codec.""" - -ShardingIndexLocation = Literal["start", "end"] -"""Literal type of the position of the shard index within the encoded shard.""" - -SHARDING_INDEX_LOCATION: Final = ("start", "end") -"""Tuple of permitted values for the `index_location` field of the `sharding_indexed` codec.""" - - -class ShardingIndexedCodecConfiguration(TypedDict, closed=True): - """ - Configuration for the Zarr v3 `sharding_indexed` codec. - - `chunk_shape` is the shape of inner chunks along each dimension; - it must evenly divide the shard shape. - - `codecs` is the codec pipeline applied to each inner chunk; exactly - one array-to-bytes codec is required. - - `index_codecs` is the codec pipeline applied to the shard index. It may - contain only codecs of static size, because a reader locates the index - by a size that it knows before reading it. A compressor in this - pipeline is therefore rejected. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L147-L155 - - `index_location` defaults to `"end"` per the spec. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L157-L161 - """ - - chunk_shape: tuple[Annotated[int, Ge(1)], ...] - """The shape of the inner chunks a shard is divided into. - - One length, at least 1, per axis of the shard the codec is handed, each - dividing the shard's length along that axis. - """ - codecs: tuple[ZarrV3CodecDeclaration, ...] - """The codec pipeline that encodes each inner chunk. - - Like an array's `codecs`, it holds exactly one array -> bytes codec. - """ - index_codecs: tuple[ZarrV3StaticCodecDeclaration, ...] - """The codec pipeline that encodes the shard index. - - It holds exactly one array -> bytes codec, and only codecs whose output - size is fixed by the size of their input, so no compressor. The spec - recommends a little-endian `bytes` codec followed by `crc32c`. - """ - index_location: NotRequired[ShardingIndexLocation] - """Whether the shard index sits at the `"start"` or the `"end"` of the encoded shard. - - Absent means `"end"`. - """ - - -class ShardingIndexedCodecObject(TypedDict, closed=True): - """`sharding_indexed` codec metadata in object form.""" - - name: ShardingIndexedCodecName - """The codec's name, `"sharding_indexed"`.""" - configuration: ShardingIndexedCodecConfiguration - """The codec's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -ShardingIndexedCodecMetadata = ShardingIndexedCodecObject -"""Permitted JSON shape for `sharding_indexed` codec metadata. - -The configuration has multiple required keys (`chunk_shape`, `codecs`, -`index_codecs`), so only the object form is valid; the short-hand-name -form is not permitted by the spec for this codec. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L129-L138 (`chunk_shape`) - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L141-L155 (`codecs` and `index_codecs`, the members the spec marks required) - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") -""" +from zarr_metadata.v3._types.codec import ( + SHARDING_INDEX_LOCATION, + SHARDING_INDEXED_CODEC_NAME, + ShardingIndexedCodecConfiguration, + ShardingIndexedCodecMetadata, + ShardingIndexedCodecName, + ShardingIndexedCodecObject, + ShardingIndexLocation, +) +from zarr_metadata.v3._types.data_type import UINT64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE def _chunk_rules( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index d32886182e..b4e34b2ab6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -5,9 +5,7 @@ """ from collections.abc import Iterator -from typing import Final, Literal, NotRequired - -from typing_extensions import TypedDict +from typing import Final from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import Nested @@ -15,54 +13,13 @@ Chunk, CodecDefinition, ) - -TRANSPOSE_CODEC_NAME: Final = "transpose" -"""The `name` field value of the `transpose` codec.""" - -TransposeCodecName = Literal["transpose"] -"""Literal type of the `name` field of the `transpose` codec.""" - - -class TransposeCodecConfiguration(TypedDict, closed=True): - """ - Configuration for the Zarr v3 `transpose` codec. - - `order` is a permutation of the dimension indices 0..n-1 that - specifies the dimension reordering applied during encoding. - """ - - order: tuple[int, ...] - """The reordering of axes applied when encoding: axis `i` of the encoded chunk is axis `order[i]` of the chunk the codec is handed. - - A permutation of `0, 1, ..., n-1`, `n` being the number of axes of the - chunk the codec is handed. - """ - - -class TransposeCodecObject(TypedDict, closed=True): - """`transpose` codec metadata in object form.""" - - name: TransposeCodecName - """The codec's name, `"transpose"`.""" - configuration: TransposeCodecConfiguration - """The codec's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -TransposeCodecMetadata = TransposeCodecObject -"""Permitted JSON shape for `transpose` codec metadata. - -`order` is required, so only the object form is valid; the short-hand-name -form is not permitted by the spec for this codec. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/transpose/index.rst#L60-L66 ("order: Required") - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") -""" +from zarr_metadata.v3._types.codec import ( + TRANSPOSE_CODEC_NAME, + TransposeCodecConfiguration, + TransposeCodecMetadata, + TransposeCodecName, + TransposeCodecObject, +) def _rules( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py index c649beefc2..8b2f42da68 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py @@ -9,46 +9,23 @@ """ from collections.abc import Iterator -from typing import Final, Literal, NotRequired - -from typing_extensions import TypedDict +from typing import Final +from zarr_metadata._common import EmptyConfiguration from zarr_metadata._json import ValidationProblem -from zarr_metadata.definition._definition import EmptyConfiguration, Nested +from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import Chunk, CodecDefinition +from zarr_metadata.v3._types.codec import ( + VLEN_BYTES_CODEC_NAME, + VlenBytesCodecMetadata, + VlenBytesCodecName, + VlenBytesCodecObject, +) +from zarr_metadata.v3._types.data_type import ( + BYTES_DATA_TYPE_NAME, + VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME, +) from zarr_metadata.v3.codec._vlen import vlen_problems -from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.variable_length_bytes import VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME - -VLEN_BYTES_CODEC_NAME: Final = "vlen-bytes" -"""The `name` field value of the `vlen-bytes` codec.""" - -VlenBytesCodecName = Literal["vlen-bytes"] -"""Literal type of the `name` field of the `vlen-bytes` codec.""" - - -class VlenBytesCodecObject(TypedDict, closed=True): - """`vlen-bytes` codec metadata in object form. The codec has no configuration fields.""" - - name: VlenBytesCodecName - """The codec's name, `"vlen-bytes"`.""" - configuration: NotRequired[EmptyConfiguration] - """The codec's parameters, of which there are none. - - Absent, or an empty mapping. - """ - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -VlenBytesCodecMetadata = VlenBytesCodecObject | VlenBytesCodecName -"""Permitted JSON shapes for `vlen-bytes` codec metadata: the object form, or the bare name, since the codec has no configuration.""" - _COMPATIBLE: Final = frozenset({BYTES_DATA_TYPE_NAME, VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME}) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py index 95fa0c5fd6..a698288646 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py @@ -9,45 +9,20 @@ """ from collections.abc import Iterator -from typing import Final, Literal, NotRequired - -from typing_extensions import TypedDict +from typing import Final +from zarr_metadata._common import EmptyConfiguration from zarr_metadata._json import ValidationProblem -from zarr_metadata.definition._definition import EmptyConfiguration, Nested +from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import Chunk, CodecDefinition +from zarr_metadata.v3._types.codec import ( + VLEN_UTF8_CODEC_NAME, + VlenUtf8CodecMetadata, + VlenUtf8CodecName, + VlenUtf8CodecObject, +) +from zarr_metadata.v3._types.data_type import STRING_DATA_TYPE_NAME from zarr_metadata.v3.codec._vlen import vlen_problems -from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE_NAME - -VLEN_UTF8_CODEC_NAME: Final = "vlen-utf8" -"""The `name` field value of the `vlen-utf8` codec.""" - -VlenUtf8CodecName = Literal["vlen-utf8"] -"""Literal type of the `name` field of the `vlen-utf8` codec.""" - - -class VlenUtf8CodecObject(TypedDict, closed=True): - """`vlen-utf8` codec metadata in object form. The codec has no configuration fields.""" - - name: VlenUtf8CodecName - """The codec's name, `"vlen-utf8"`.""" - configuration: NotRequired[EmptyConfiguration] - """The codec's parameters, of which there are none. - - Absent, or an empty mapping. - """ - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -VlenUtf8CodecMetadata = VlenUtf8CodecObject | VlenUtf8CodecName -"""Permitted JSON shapes for `vlen-utf8` codec metadata: the object form, or the bare name, since the codec has no configuration.""" - _COMPATIBLE: Final = frozenset({STRING_DATA_TYPE_NAME}) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 222346caac..a4153ca6f8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -6,73 +6,18 @@ proposed the codec, was never merged). """ -from typing import Annotated, Final, Literal, NotRequired - -from annotated_types import Interval -from typing_extensions import TypedDict +from typing import Final from zarr_metadata.v3._definition import CodecDefinition - -ZSTD_CODEC_NAME: Final = "zstd" -"""The `name` field value of the `zstd` codec.""" - -ZstdCodecName = Literal["zstd"] -"""Literal type of the `name` field of the `zstd` codec.""" - -ZSTD_MIN_LEVEL: Final = -131072 -"""The lowest `level` zstd accepts: ZSTD_minCLevel(), -(1 << 17).""" - -ZSTD_MAX_LEVEL: Final = 22 -"""The highest `level` zstd accepts: ZSTD_maxCLevel().""" - - -class ZstdCodecConfiguration(TypedDict, closed=True): - """ - Configuration for the Zarr v3 `zstd` codec. - - `level` is required; `checksum` is optional ("Should be omitted if - false"). - https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 - """ - - level: Annotated[int, Interval(ge=ZSTD_MIN_LEVEL, le=ZSTD_MAX_LEVEL)] - """The compression level, an integer from `ZSTD_MIN_LEVEL` (-131072) to `ZSTD_MAX_LEVEL` (22). - - 0 means zstd's default level; otherwise a higher level compresses more, - more slowly. It has no effect on decoding. - """ - checksum: NotRequired[bool] - """Whether a checksum is stored when writing and verified when reading. - - Absent means `false`; the registry says to omit it rather than write - `false`. - """ - - -class ZstdCodecObject(TypedDict, closed=True): - """`zstd` codec metadata in object form.""" - - name: ZstdCodecName - """The codec's name, `"zstd"`.""" - configuration: ZstdCodecConfiguration - """The codec's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this codec must refuse the document. - - Absent means `true`. The spec permits `false` for a codec - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), - though `resolve` reports a `false` as a problem at every extension point. - """ - - -ZstdCodecMetadata = ZstdCodecObject -"""Permitted JSON shape for `zstd` codec metadata. - -`level` is required, so only the object form is valid; the short-hand-name -form is not permitted by the spec for this codec. - https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") -""" +from zarr_metadata.v3._types.codec import ( + ZSTD_CODEC_NAME, + ZSTD_MAX_LEVEL, + ZSTD_MIN_LEVEL, + ZstdCodecConfiguration, + ZstdCodecMetadata, + ZstdCodecName, + ZstdCodecObject, +) def _canonical(configuration: ZstdCodecConfiguration) -> ZstdCodecConfiguration: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index 7d3344cf01..4170a8d454 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -13,77 +13,14 @@ the entries as the reference implementation writes them. """ -from collections.abc import Mapping -from typing import Any, Final, Literal +from typing import Any, Final -from typing_extensions import TypedDict - -from zarr_metadata._common import UserJSONValue from zarr_metadata.v3._definition import GroupExtraFieldDefinition -from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON -from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON - - -class ZarrV3ConsolidatedMetadataEnvelopeJSON(TypedDict, closed=True): - """ - The `consolidated_metadata` member of a v3 group document, as the core spec fixes it. - - The spec allows this one additional field on a group, for historical - reasons, and fixes its envelope: `kind` is `"inline"`, - `must_understand` is false, and `metadata` is an object. The spec - leaves the entries of `metadata` to the reference implementation, so - here they are any JSON, as user data: the attributes of a listed - document may hold `NaN`, `Infinity` and `-Infinity`. - `ZarrV3ConsolidatedMetadataJSON` types the - entries as the reference implementation writes them: an array or group - document at each path. - - See https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 - """ - - kind: Literal["inline"] - """How the entries are carried, `"inline"`: in the group's own document.""" - must_understand: Literal[False] - """Always `false`: a reader that does not recognize consolidated metadata may ignore it.""" - metadata: Mapping[str, UserJSONValue] - """The entries, by each node's path from the group, as any JSON.""" - - -class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): - """ - Inline consolidated metadata embedded in a v3 group. - - The `metadata` map contains only v3 array and group entries. V2 entries - are excluded from this interoperability convention by design. The v3 core - specification acknowledges `consolidated_metadata` as a historical - additional field and fixes this envelope, but leaves the entries to the - reference implementation: - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 - """ - - kind: Literal["inline"] - """How the entries are carried, `"inline"`: in the group's own document.""" - must_understand: Literal[False] - """Always `false`: a reader that does not recognize consolidated metadata may ignore it.""" - metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] - """The metadata document of each node below the group, by its path from the group. - - A path joins its segments with `"/"`, e.g. `"child/grandchild"`. - """ - - -ZARR_V3_CONSOLIDATED_METADATA_KEY: Final = "consolidated_metadata" -"""The key under which consolidated metadata is embedded in a v3 group document. - -Unlike the v2 `.zmetadata` file, this is not a store key: consolidated metadata -is carried as an additional field inside the group's own `zarr.json`. The core -spec names the field and its envelope ("For historical reasons, group metadata -documents may contain an additional field named ``consolidated_metadata``"); -the entry format, like the v2 counterpart, is a reference-implementation -convention. - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 -""" - +from zarr_metadata.v3._types.consolidated import ( + ZARR_V3_CONSOLIDATED_METADATA_KEY, + ZarrV3ConsolidatedMetadataEnvelopeJSON, + ZarrV3ConsolidatedMetadataJSON, +) CONSOLIDATED_METADATA_EXTRA_FIELD: Final[GroupExtraFieldDefinition[Any]] = ( GroupExtraFieldDefinition( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_byte.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_byte.py index f9eda83e16..b20fb9a7db 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_byte.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_byte.py @@ -1,22 +1,15 @@ -"""A byte value, as used in the fill values of raw bits and of `bytes`, and the rules of a fill value written in base64. +"""The rules of a fill value written in base64, which the data types whose fill values are bytes share. -Each byte value is an integer in `[0, 255]` -(https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L97-L99; -the reference for the `bytes` type is https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md?plain=1#L8). +The type of one byte of such a fill value, `ByteValue`, is in +`zarr_metadata.v3._types.data_type`. """ import base64 from collections.abc import Iterator -from typing import Annotated - -from annotated_types import Interval from zarr_metadata._json import ValidationProblem, shown from zarr_metadata._numpy import base64_bytes -ByteValue = Annotated[int, Interval(ge=0, le=255)] -"""One byte of a fill value: a JSON integer in `[0, 255]`.""" - def base64_problems(value: str) -> Iterator[ValidationProblem]: """Check that a fill value written as a string is standard-alphabet base64.""" @@ -59,7 +52,6 @@ def base64_spelling(data: bytes) -> str: __all__ = [ - "ByteValue", "base64_decoded", "base64_fill_value_rules", "base64_problems", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_float.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_float.py index 3cfba1404e..c291eb2a52 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_float.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_float.py @@ -16,22 +16,12 @@ from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass from decimal import ROUND_HALF_EVEN, Decimal, localcontext -from typing import Final, Literal, get_args +from typing import Final, Literal -from zarr_metadata._common import JSONValue +from zarr_metadata._common import EmptyConfiguration, JSONValue from zarr_metadata._json import ValidationProblem, choices, shown -from zarr_metadata.definition._definition import ( - EmptyConfiguration, - Nested, -) - -FloatSpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] -"""The named non-finite fill values every IEEE 754 floating-point type takes.""" - -FLOAT_SPECIAL_FILL_VALUES: Final[tuple[FloatSpecialFillValue, ...]] = get_args( - FloatSpecialFillValue -) -"""The strings that the fill value rules accept for a float fill value in place of a number.""" +from zarr_metadata.definition._definition import Nested +from zarr_metadata.v3._types.data_type import FLOAT_SPECIAL_FILL_VALUES def float_fill_value_rules( @@ -246,7 +236,6 @@ def __call__( __all__ = [ - "FloatSpecialFillValue", "FloatWidth", "complex_fill_value_canonical", "complex_fill_value_rules", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_numpy_time.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_numpy_time.py index ed631a353f..92dc70d91e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_numpy_time.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_numpy_time.py @@ -1,22 +1,14 @@ -"""The time units of the two numpy time types and how they store their values, defined once so that neither type module imports them from the other. +"""How the two numpy time types store their values, defined once so that neither type module imports it from the other. -The encodings both types share with the v2 time dtypes are in +Their time units are in `zarr_metadata.v3._types.data_type`, and the +encodings both types share with the v2 time dtypes are in `zarr_metadata._numpy`. """ -from typing import Final, Literal +from typing import Final -from zarr_metadata._numpy import NUMPY_TIME_UNITS from zarr_metadata.v3._definition import multi_byte -NumpyTimeUnit = Literal[ - "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" -] -"""The time unit codes of the numpy time types, `numpy.datetime64` and `numpy.timedelta64`.""" - -NUMPY_TIME_UNIT: Final = NUMPY_TIME_UNITS -"""Runtime tuple of the permitted `numpy.timedelta64`/`numpy.datetime64` unit strings.""" - numpy_time_storage: Final = multi_byte """The storage of the numpy time types: signed 64-bit integers, in the byte order that the codecs specify. @@ -30,4 +22,4 @@ """ -__all__ = ["NUMPY_TIME_UNIT", "NumpyTimeUnit", "numpy_time_storage"] +__all__ = ["numpy_time_storage"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index abc5395e1b..3b12c2477d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -4,23 +4,14 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from typing import Final -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, single_byte, ) - -BOOL_DATA_TYPE_NAME: Final = "bool" -"""The `data_type` value for the `bool` type.""" - -BoolDataTypeName = Literal["bool"] -"""Literal type of the `data_type` field for `bool`.""" - -BoolFillValue = bool -"""Permitted JSON shape of the `fill_value` field for `bool`: a JSON boolean.""" - +from zarr_metadata.v3._types.data_type import BOOL_DATA_TYPE_NAME, BoolDataTypeName, BoolFillValue BOOL_DATA_TYPE: Final = DataTypeDefinition( name=BOOL_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index f7e5c572aa..6f387d4fea 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -5,35 +5,21 @@ """ import base64 -from typing import Final, Literal +from typing import Final +from zarr_metadata._common import EmptyConfiguration from zarr_metadata._numpy import Base64Bytes, base64_bytes -from zarr_metadata.definition._definition import ( - EmptyConfiguration, - Nested, -) +from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import ( DataTypeDefinition, variable_length, ) -from zarr_metadata.v3.data_type._byte import ( - ByteValue, - base64_fill_value_rules, - base64_spelling, +from zarr_metadata.v3._types.data_type import ( + BYTES_DATA_TYPE_NAME, + BytesDataTypeName, + BytesFillValue, ) - -BYTES_DATA_TYPE_NAME: Final = "bytes" -"""The `data_type` value for the variable-length `bytes` type.""" - -BytesDataTypeName = Literal["bytes"] -"""Literal type of the `data_type` field for `bytes`.""" - -BytesFillValue = tuple[ByteValue, ...] | Base64Bytes -"""Permitted JSON shape of the `fill_value` field for `bytes`. - -Either a JSON array of integers in `[0, 255]` (one per byte), or a -`Base64Bytes` string encoding the byte sequence. -""" +from zarr_metadata.v3.data_type._byte import base64_fill_value_rules, base64_spelling def _fill_value_canonical( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py index 8d7bfefb69..d58a6d941f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py @@ -4,39 +4,24 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from typing import Final -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) +from zarr_metadata.v3._types.data_type import ( + COMPLEX128_DATA_TYPE_NAME, + Complex128Component, + Complex128DataTypeName, + Complex128FillValue, +) from zarr_metadata.v3.data_type._float import ( complex_fill_value_canonical, complex_fill_value_rules, ) -from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE, Float64FillValue - -COMPLEX128_DATA_TYPE_NAME: Final = "complex128" -"""The `data_type` value for the `complex128` type.""" - -Complex128DataTypeName = Literal["complex128"] -"""Literal type of the `data_type` field for `complex128`.""" - -Complex128Component = Float64FillValue -"""One real or imaginary component of a `complex128` fill value. - -Same shape as a `float64` fill value: a JSON number, a named sentinel, -or a `HexFloat64` string. -""" - -Complex128FillValue = tuple[Complex128Component, Complex128Component] -"""Permitted JSON shape of the `fill_value` field for `complex128`. - -A two-element JSON array `[real, imag]` where each component is a -`Complex128Component`. -""" - +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE COMPLEX128_DATA_TYPE: Final = DataTypeDefinition( name=COMPLEX128_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py index 4024af79b8..41b6e206db 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py @@ -4,39 +4,24 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from typing import Final -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) +from zarr_metadata.v3._types.data_type import ( + COMPLEX64_DATA_TYPE_NAME, + Complex64Component, + Complex64DataTypeName, + Complex64FillValue, +) from zarr_metadata.v3.data_type._float import ( complex_fill_value_canonical, complex_fill_value_rules, ) -from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE, Float32FillValue - -COMPLEX64_DATA_TYPE_NAME: Final = "complex64" -"""The `data_type` value for the `complex64` type.""" - -Complex64DataTypeName = Literal["complex64"] -"""Literal type of the `data_type` field for `complex64`.""" - -Complex64Component = Float32FillValue -"""One real or imaginary component of a `complex64` fill value. - -Same shape as a `float32` fill value: a JSON number, a named sentinel, -or a `HexFloat32` string. -""" - -Complex64FillValue = tuple[Complex64Component, Complex64Component] -"""Permitted JSON shape of the `fill_value` field for `complex64`. - -A two-element JSON array `[real, imag]` where each component is a -`Complex64Component`. -""" - +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE COMPLEX64_DATA_TYPE: Final = DataTypeDefinition( name=COMPLEX64_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py index 35705f56f5..24e3a25443 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py @@ -5,50 +5,22 @@ """ from collections.abc import Iterator -from typing import Final, Literal, NotRequired - -from typing_extensions import ReadOnly, TypedDict +from typing import Final from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import DataTypeDefinition, multi_byte - -FIXED_LENGTH_UTF32_DATA_TYPE_NAME: Final = "fixed_length_utf32" -"""The `name` field value of the `fixed_length_utf32` data type.""" - -FixedLengthUtf32DataTypeName = Literal["fixed_length_utf32"] -"""Literal type of the `name` field of the `fixed_length_utf32` data type.""" +from zarr_metadata.v3._types.data_type import ( + FIXED_LENGTH_UTF32_DATA_TYPE_NAME, + FixedLengthUtf32, + FixedLengthUtf32Configuration, + FixedLengthUtf32DataTypeName, + FixedLengthUtf32FillValue, +) _CODE_POINT_BYTES: Final = 4 -class FixedLengthUtf32Configuration(TypedDict, closed=True): - """Configuration for the `fixed_length_utf32` data type.""" - - length_bytes: ReadOnly[int] - """The size in bytes of every value: a positive multiple of 4, since - each code point takes 4 bytes.""" - - -class FixedLengthUtf32(TypedDict, closed=True): - """`fixed_length_utf32` data type metadata.""" - - name: FixedLengthUtf32DataTypeName - """The data type's name, `"fixed_length_utf32"`.""" - configuration: FixedLengthUtf32Configuration - """The data type's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this data type must refuse the document. - - Absent means `true`. The spec does not permit `false` for a data type - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). - """ - - -FixedLengthUtf32FillValue = str -"""Permitted JSON shape of the `fill_value` field for `fixed_length_utf32`: a JSON string of at most `length_bytes / 4` code points.""" - - def _rules( configuration: FixedLengthUtf32Configuration, nested: Nested ) -> Iterator[ValidationProblem]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 33dbf599ca..bd5eb54f01 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -5,34 +5,24 @@ """ import re -from typing import Final, Literal, NewType +from typing import Final -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) -from zarr_metadata.v3.data_type._float import ( - FloatSpecialFillValue, - float_fill_value_canonical, - float_fill_value_rules, +from zarr_metadata.v3._types.data_type import ( + CANONICAL_NAN_HEX_FLOAT16, + CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT16, + CANONICAL_POSITIVE_INFINITY_HEX_FLOAT16, + FLOAT16_DATA_TYPE_NAME, + Float16DataTypeName, + Float16FillValue, + Float16SpecialFillValue, + HexFloat16, ) - -FLOAT16_DATA_TYPE_NAME: Final = "float16" -"""The `data_type` value for the `float16` type.""" - -Float16DataTypeName = Literal["float16"] -"""Literal type of the `data_type` field for `float16`.""" - -Float16SpecialFillValue = FloatSpecialFillValue -"""Named non-finite fill values permitted by the spec for IEEE 754 floats. - -https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 -""" - -HexFloat16 = NewType("HexFloat16", str) -"""A 6-character hex string (`0x` + 4 hex digits) encoding the -unsigned-integer representation of a float16.""" +from zarr_metadata.v3.data_type._float import float_fill_value_canonical, float_fill_value_rules _HEX_FLOAT16_RE: Final = re.compile(r"^0x[0-9a-fA-F]{4}$") @@ -48,30 +38,6 @@ def hex_float16(value: str) -> HexFloat16: return HexFloat16(value) -Float16FillValue = float | int | Float16SpecialFillValue | HexFloat16 -"""Permitted JSON shape of the `fill_value` field for `float16`. - -Either a JSON number, one of the named non-finite sentinels (`"NaN"`, -`"Infinity"`, `"-Infinity"`), or a `HexFloat16` (`0xYYYY` string encoding -the unsigned-integer representation of the IEEE 754 value). -""" - -CANONICAL_NAN_HEX_FLOAT16: Final = "0x7e00" -"""Canonical hex form of the float16 NaN sentinel `"NaN"`. - -Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the -most significant mantissa bit set, and all other mantissa bits zero -(the IEEE 754 default quiet NaN). Other NaN bit patterns must be -encoded with the explicit hex-string form. -""" - -CANONICAL_POSITIVE_INFINITY_HEX_FLOAT16: Final = "0x7c00" -"""Canonical hex form of the float16 `"Infinity"` sentinel.""" - -CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT16: Final = "0xfc00" -"""Canonical hex form of the float16 `"-Infinity"` sentinel.""" - - FLOAT16_DATA_TYPE: Final = DataTypeDefinition( name=FLOAT16_DATA_TYPE_NAME, configuration=EmptyConfiguration, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index b4760426a7..3e391d8cb5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -5,34 +5,24 @@ """ import re -from typing import Final, Literal, NewType +from typing import Final -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) -from zarr_metadata.v3.data_type._float import ( - FloatSpecialFillValue, - float_fill_value_canonical, - float_fill_value_rules, +from zarr_metadata.v3._types.data_type import ( + CANONICAL_NAN_HEX_FLOAT32, + CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT32, + CANONICAL_POSITIVE_INFINITY_HEX_FLOAT32, + FLOAT32_DATA_TYPE_NAME, + Float32DataTypeName, + Float32FillValue, + Float32SpecialFillValue, + HexFloat32, ) - -FLOAT32_DATA_TYPE_NAME: Final = "float32" -"""The `data_type` value for the `float32` type.""" - -Float32DataTypeName = Literal["float32"] -"""Literal type of the `data_type` field for `float32`.""" - -Float32SpecialFillValue = FloatSpecialFillValue -"""Named non-finite fill values permitted by the spec for IEEE 754 floats. - -https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 -""" - -HexFloat32 = NewType("HexFloat32", str) -"""A 10-character hex string (`0x` + 8 hex digits) encoding the -unsigned-integer representation of a float32.""" +from zarr_metadata.v3.data_type._float import float_fill_value_canonical, float_fill_value_rules _HEX_FLOAT32_RE: Final = re.compile(r"^0x[0-9a-fA-F]{8}$") @@ -48,30 +38,6 @@ def hex_float32(value: str) -> HexFloat32: return HexFloat32(value) -Float32FillValue = float | int | Float32SpecialFillValue | HexFloat32 -"""Permitted JSON shape of the `fill_value` field for `float32`. - -Either a JSON number, one of the named non-finite sentinels (`"NaN"`, -`"Infinity"`, `"-Infinity"`), or a `HexFloat32` (`0xYYYYYYYY` string -encoding the unsigned-integer representation of the IEEE 754 value). -""" - -CANONICAL_NAN_HEX_FLOAT32: Final = "0x7fc00000" -"""Canonical hex form of the float32 NaN sentinel `"NaN"`. - -Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the -most significant mantissa bit set, and all other mantissa bits zero -(the IEEE 754 default quiet NaN). Other NaN bit patterns must be -encoded with the explicit hex-string form. -""" - -CANONICAL_POSITIVE_INFINITY_HEX_FLOAT32: Final = "0x7f800000" -"""Canonical hex form of the float32 `"Infinity"` sentinel.""" - -CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT32: Final = "0xff800000" -"""Canonical hex form of the float32 `"-Infinity"` sentinel.""" - - FLOAT32_DATA_TYPE: Final = DataTypeDefinition( name=FLOAT32_DATA_TYPE_NAME, configuration=EmptyConfiguration, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index cf9f91c67e..83b73ee62f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -5,34 +5,24 @@ """ import re -from typing import Final, Literal, NewType +from typing import Final -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) -from zarr_metadata.v3.data_type._float import ( - FloatSpecialFillValue, - float_fill_value_canonical, - float_fill_value_rules, +from zarr_metadata.v3._types.data_type import ( + CANONICAL_NAN_HEX_FLOAT64, + CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT64, + CANONICAL_POSITIVE_INFINITY_HEX_FLOAT64, + FLOAT64_DATA_TYPE_NAME, + Float64DataTypeName, + Float64FillValue, + Float64SpecialFillValue, + HexFloat64, ) - -FLOAT64_DATA_TYPE_NAME: Final = "float64" -"""The `data_type` value for the `float64` type.""" - -Float64DataTypeName = Literal["float64"] -"""Literal type of the `data_type` field for `float64`.""" - -Float64SpecialFillValue = FloatSpecialFillValue -"""Named non-finite fill values permitted by the spec for IEEE 754 floats. - -https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 -""" - -HexFloat64 = NewType("HexFloat64", str) -"""An 18-character hex string (`0x` + 16 hex digits) encoding the -unsigned-integer representation of a float64.""" +from zarr_metadata.v3.data_type._float import float_fill_value_canonical, float_fill_value_rules _HEX_FLOAT64_RE: Final = re.compile(r"^0x[0-9a-fA-F]{16}$") @@ -48,31 +38,6 @@ def hex_float64(value: str) -> HexFloat64: return HexFloat64(value) -Float64FillValue = float | int | Float64SpecialFillValue | HexFloat64 -"""Permitted JSON shape of the `fill_value` field for `float64`. - -Either a JSON number, one of the named non-finite sentinels (`"NaN"`, -`"Infinity"`, `"-Infinity"`), or a `HexFloat64` (`0xYYYYYYYYYYYYYYYY` -string encoding the unsigned-integer representation of the IEEE 754 -value). -""" - -CANONICAL_NAN_HEX_FLOAT64: Final = "0x7ff8000000000000" -"""Canonical hex form of the float64 NaN sentinel `"NaN"`. - -Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the -most significant mantissa bit set, and all other mantissa bits zero -(the IEEE 754 default quiet NaN). Other NaN bit patterns must be -encoded with the explicit hex-string form. -""" - -CANONICAL_POSITIVE_INFINITY_HEX_FLOAT64: Final = "0x7ff0000000000000" -"""Canonical hex form of the float64 `"Infinity"` sentinel.""" - -CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT64: Final = "0xfff0000000000000" -"""Canonical hex form of the float64 `"-Infinity"` sentinel.""" - - FLOAT64_DATA_TYPE: Final = DataTypeDefinition( name=FLOAT64_DATA_TYPE_NAME, configuration=EmptyConfiguration, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py index cf7d6bfce3..53c0982a33 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py @@ -4,25 +4,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Annotated, Final, Literal +from typing import Final -from annotated_types import Interval - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) - -INT16_DATA_TYPE_NAME: Final = "int16" -"""The `data_type` value for the `int16` type.""" - -Int16DataTypeName = Literal["int16"] -"""Literal type of the `data_type` field for `int16`.""" - -Int16FillValue = Annotated[int, Interval(ge=-(2**15), le=2**15 - 1)] -"""Permitted JSON shape of the `fill_value` field for `int16`: a JSON integer in [-32768, 32767].""" - +from zarr_metadata.v3._types.data_type import ( + INT16_DATA_TYPE_NAME, + Int16DataTypeName, + Int16FillValue, +) INT16_DATA_TYPE: Final = DataTypeDefinition( name=INT16_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py index 7259e53bc7..8c8e204404 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py @@ -4,25 +4,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Annotated, Final, Literal +from typing import Final -from annotated_types import Interval - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) - -INT32_DATA_TYPE_NAME: Final = "int32" -"""The `data_type` value for the `int32` type.""" - -Int32DataTypeName = Literal["int32"] -"""Literal type of the `data_type` field for `int32`.""" - -Int32FillValue = Annotated[int, Interval(ge=-(2**31), le=2**31 - 1)] -"""Permitted JSON shape of the `fill_value` field for `int32`: a JSON integer in [-2**31, 2**31 - 1].""" - +from zarr_metadata.v3._types.data_type import ( + INT32_DATA_TYPE_NAME, + Int32DataTypeName, + Int32FillValue, +) INT32_DATA_TYPE: Final = DataTypeDefinition( name=INT32_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py index 259243572b..48ba266f66 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py @@ -4,25 +4,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Annotated, Final, Literal +from typing import Final -from annotated_types import Interval - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) - -INT64_DATA_TYPE_NAME: Final = "int64" -"""The `data_type` value for the `int64` type.""" - -Int64DataTypeName = Literal["int64"] -"""Literal type of the `data_type` field for `int64`.""" - -Int64FillValue = Annotated[int, Interval(ge=-(2**63), le=2**63 - 1)] -"""Permitted JSON shape of the `fill_value` field for `int64`: a JSON integer in [-2**63, 2**63 - 1].""" - +from zarr_metadata.v3._types.data_type import ( + INT64_DATA_TYPE_NAME, + Int64DataTypeName, + Int64FillValue, +) INT64_DATA_TYPE: Final = DataTypeDefinition( name=INT64_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py index 98f321edfa..75b986e9ca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py @@ -4,25 +4,14 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Annotated, Final, Literal +from typing import Final -from annotated_types import Interval - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, single_byte, ) - -INT8_DATA_TYPE_NAME: Final = "int8" -"""The `data_type` value for the `int8` type.""" - -Int8DataTypeName = Literal["int8"] -"""Literal type of the `data_type` field for `int8`.""" - -Int8FillValue = Annotated[int, Interval(ge=-(2**7), le=2**7 - 1)] -"""Permitted JSON shape of the `fill_value` field for `int8`: a JSON integer in [-128, 127].""" - +from zarr_metadata.v3._types.data_type import INT8_DATA_TYPE_NAME, Int8DataTypeName, Int8FillValue INT8_DATA_TYPE: Final = DataTypeDefinition( name=INT8_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py index cbd61781db..7bde2bce1d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py @@ -10,48 +10,20 @@ import base64 from collections.abc import Iterator -from typing import Final, Literal, NotRequired - -from typing_extensions import ReadOnly, TypedDict +from typing import Final from zarr_metadata._json import ValidationProblem -from zarr_metadata._numpy import Base64Bytes from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import DataTypeDefinition, single_byte +from zarr_metadata.v3._types.data_type import ( + NULL_TERMINATED_BYTES_DATA_TYPE_NAME, + NullTerminatedBytes, + NullTerminatedBytesConfiguration, + NullTerminatedBytesDataTypeName, + NullTerminatedBytesFillValue, +) from zarr_metadata.v3.data_type._byte import base64_decoded, base64_problems, base64_spelling -NULL_TERMINATED_BYTES_DATA_TYPE_NAME: Final = "null_terminated_bytes" -"""The `name` field value of the `null_terminated_bytes` data type.""" - -NullTerminatedBytesDataTypeName = Literal["null_terminated_bytes"] -"""Literal type of the `name` field of the `null_terminated_bytes` data type.""" - - -class NullTerminatedBytesConfiguration(TypedDict, closed=True): - """Configuration for the `null_terminated_bytes` data type.""" - - length_bytes: ReadOnly[int] - """The size in bytes of every value, at least 1.""" - - -class NullTerminatedBytes(TypedDict, closed=True): - """`null_terminated_bytes` data type metadata.""" - - name: NullTerminatedBytesDataTypeName - """The data type's name, `"null_terminated_bytes"`.""" - configuration: NullTerminatedBytesConfiguration - """The data type's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this data type must refuse the document. - - Absent means `true`. The spec does not permit `false` for a data type - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). - """ - - -NullTerminatedBytesFillValue = Base64Bytes -"""Permitted JSON shape of the `fill_value` field for `null_terminated_bytes`: the bytes, without their trailing null bytes, as a `Base64Bytes` string.""" - def _rules( configuration: NullTerminatedBytesConfiguration, nested: Nested diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index af4394abb9..376f70a0e1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -4,66 +4,22 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md """ -from typing import Final, Literal, NotRequired - -from typing_extensions import ReadOnly, TypedDict +from typing import Final from zarr_metadata._numpy import ( - NumpyTimeScaleFactor, - NumpyTimeTicks, numpy_time_fill_value_canonical, ) from zarr_metadata.v3._definition import DataTypeDefinition -from zarr_metadata.v3.data_type._numpy_time import ( +from zarr_metadata.v3._types.data_type import ( + NUMPY_DATETIME64_DATA_TYPE_NAME, NUMPY_TIME_UNIT, + NumpyDatetime64, + NumpyDatetime64Configuration, + NumpyDatetime64DataTypeName, + NumpyDatetime64FillValue, NumpyTimeUnit, - numpy_time_storage, ) - -NUMPY_DATETIME64_DATA_TYPE_NAME: Final = "numpy.datetime64" -"""The `name` field value of the `numpy.datetime64` data type.""" - -NumpyDatetime64DataTypeName = Literal["numpy.datetime64"] -"""Literal type of the `name` field of the `numpy.datetime64` data type.""" - - -class NumpyDatetime64Configuration(TypedDict, closed=True): - """Configuration for the `numpy.datetime64` data type.""" - - unit: ReadOnly[NumpyTimeUnit] - """A string encoding a unit of time. - - `"us"` and `"μs"` are equivalent spellings of microseconds; `"generic"` - means the unit is unspecified. - """ - scale_factor: ReadOnly[NumpyTimeScaleFactor] - """The multiplier relative to the unit. - - An integer in `[1, 2147483647]`. - """ - - -class NumpyDatetime64(TypedDict, closed=True): - """`numpy.datetime64` data type metadata.""" - - name: NumpyDatetime64DataTypeName - """The data type's name, `"numpy.datetime64"`.""" - configuration: NumpyDatetime64Configuration - """The data type's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this data type must refuse the document. - - Absent means `true`. The spec does not permit `false` for a data type - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). - """ - - -NumpyDatetime64FillValue = NumpyTimeTicks | Literal["NaT"] -"""Permitted JSON shape of the `fill_value` field for `numpy.datetime64`. - -Either a JSON integer (count of `unit * scale_factor` since the epoch), -or the string `"NaT"` (equivalent to the integer `-2**63`). -""" +from zarr_metadata.v3.data_type._numpy_time import numpy_time_storage NUMPY_DATETIME64_DATA_TYPE: Final = DataTypeDefinition( name=NUMPY_DATETIME64_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index f320226ee8..b7fd8bc555 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -4,62 +4,20 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.timedelta64/README.md """ -from typing import Final, Literal, NotRequired - -from typing_extensions import ReadOnly, TypedDict +from typing import Final from zarr_metadata._numpy import ( - NumpyTimeScaleFactor, - NumpyTimeTicks, numpy_time_fill_value_canonical, ) from zarr_metadata.v3._definition import DataTypeDefinition -from zarr_metadata.v3.data_type._numpy_time import NumpyTimeUnit, numpy_time_storage - -NUMPY_TIMEDELTA64_DATA_TYPE_NAME: Final = "numpy.timedelta64" -"""The `name` field value of the `numpy.timedelta64` data type.""" - -NumpyTimedelta64DataTypeName = Literal["numpy.timedelta64"] -"""Literal type of the `name` field of the `numpy.timedelta64` data type.""" - - -class NumpyTimedelta64Configuration(TypedDict, closed=True): - """Configuration for the `numpy.timedelta64` data type.""" - - unit: ReadOnly[NumpyTimeUnit] - """A string encoding a unit of time. - - `"us"` and `"μs"` are equivalent spellings of microseconds; `"generic"` - means the unit is unspecified. - """ - scale_factor: ReadOnly[NumpyTimeScaleFactor] - """The multiplier relative to the unit. - - An integer in `[1, 2147483647]`. - """ - - -class NumpyTimedelta64(TypedDict, closed=True): - """`numpy.timedelta64` data type metadata.""" - - name: NumpyTimedelta64DataTypeName - """The data type's name, `"numpy.timedelta64"`.""" - configuration: NumpyTimedelta64Configuration - """The data type's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this data type must refuse the document. - - Absent means `true`. The spec does not permit `false` for a data type - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). - """ - - -NumpyTimedelta64FillValue = NumpyTimeTicks | Literal["NaT"] -"""Permitted JSON shape of the `fill_value` field for `numpy.timedelta64`. - -Either a JSON integer (a count of `unit * scale_factor`), or the string -`"NaT"` (equivalent to the integer `-2**63`). -""" +from zarr_metadata.v3._types.data_type import ( + NUMPY_TIMEDELTA64_DATA_TYPE_NAME, + NumpyTimedelta64, + NumpyTimedelta64Configuration, + NumpyTimedelta64DataTypeName, + NumpyTimedelta64FillValue, +) +from zarr_metadata.v3.data_type._numpy_time import numpy_time_storage NUMPY_TIMEDELTA64_DATA_TYPE: Final = DataTypeDefinition( name=NUMPY_TIMEDELTA64_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 7e90e3fdc4..a6bf51b9c3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -9,9 +9,7 @@ """ from collections.abc import Iterator -from typing import Final, NewType - -from typing_extensions import TypedDict +from typing import Final from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested @@ -21,14 +19,11 @@ DataTypeDefinition, StorageClass, ) -from zarr_metadata.v3.data_type._byte import ByteValue - -RawBytesDataTypeName = NewType("RawBytesDataTypeName", str) -"""A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`). - -"raw bits, variable size given by *, limited to be a multiple of 8": - https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47 -""" +from zarr_metadata.v3._types.data_type import ( + RawBytesConfiguration, + RawBytesDataTypeName, + RawBytesFillValue, +) def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: @@ -46,28 +41,6 @@ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: return RawBytesDataTypeName(value) -RawBytesFillValue = tuple[ByteValue, ...] -"""Permitted JSON shape of the `fill_value` field for `r`. - -A JSON array of N/8 integers in `[0, 255]` (one per byte). -""" - - -class RawBytesConfiguration(TypedDict, closed=True): - """The configuration that a raw-bytes name carries: its size in bits, such as `{"bits": 16}` for `r16`. - - A document writes the name, never this object. The definition of `r*` - reads a name into this object, and the simplest spelling writes it - back as the name. - """ - - bits: int - """The size in bits that the name writes, `16` for `r16`. - - A positive multiple of 8. - """ - - def _rules(configuration: RawBytesConfiguration, nested: Nested) -> Iterator[ValidationProblem]: """Check the size: "raw bits, variable size given by *, limited to be a multiple of 8". Zero bits is not a valid type.""" bits = configuration["bits"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index cd341c99df..4e4512aa0e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -4,23 +4,18 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/string/README.md """ -from typing import Final, Literal +from typing import Final -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, variable_length, ) - -STRING_DATA_TYPE_NAME: Final = "string" -"""The `data_type` value for the `string` type.""" - -StringDataTypeName = Literal["string"] -"""Literal type of the `data_type` field for `string`.""" - -StringFillValue = str -"""Permitted JSON shape of the `fill_value` field for `string`: a JSON unicode string.""" - +from zarr_metadata.v3._types.data_type import ( + STRING_DATA_TYPE_NAME, + StringDataTypeName, + StringFillValue, +) STRING_DATA_TYPE: Final = DataTypeDefinition( name=STRING_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 1d6ddba1a7..0e611f3f35 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -4,10 +4,8 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md """ -from collections.abc import Iterator, Mapping -from typing import Final, Literal, NotRequired - -from typing_extensions import ReadOnly, TypedDict +from collections.abc import Iterator +from typing import Final from zarr_metadata._common import JSONValue from zarr_metadata._json import ValidationProblem, shown @@ -16,65 +14,19 @@ fill_value_problems, spelled_canonically, ) -from zarr_metadata.v3._common import ZarrV3DataTypeDeclaration from zarr_metadata.v3._definition import ( DataTypeDefinition, StorageClass, storage_of, ) - -STRUCT_DATA_TYPE_NAME: Final = "struct" -"""The `name` field value of the `struct` data type.""" - -StructDataTypeName = Literal["struct"] -"""Literal type of the `name` field of the `struct` data type.""" - - -class StructField(TypedDict, closed=True): - """A single field entry inside a structured dtype.""" - - name: ReadOnly[str] - """The field name (must be unique within a struct and non-empty).""" - data_type: ReadOnly[ZarrV3DataTypeDeclaration] - """The field's data type. - - Recursive: may be a bare-string primitive or a named-config envelope - including another `struct`. - """ - - -class StructConfiguration(TypedDict, closed=True): - """Configuration for the `struct` data type.""" - - fields: ReadOnly[tuple[StructField, ...]] - """The record's fields, in the order they are encoded. - - At least one. The order is significant: it is the order in which the - `bytes` codec packs the fields' values. - """ - - -class Struct(TypedDict, closed=True): - """`struct` data type metadata.""" - - name: StructDataTypeName - """The data type's name, `"struct"`.""" - configuration: StructConfiguration - """The data type's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this data type must refuse the document. - - Absent means `true`. The spec does not permit `false` for a data type - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). - """ - - -StructFillValue = Mapping[str, JSONValue] -"""Permitted JSON shape of the `fill_value` field for `struct`. - -A JSON object mapping each field name to that field's fill value. Field -fill values are themselves shaped per the field's `data_type`, recursively. -""" +from zarr_metadata.v3._types.data_type import ( + STRUCT_DATA_TYPE_NAME, + Struct, + StructConfiguration, + StructDataTypeName, + StructField, + StructFillValue, +) def _rules(configuration: StructConfiguration, nested: Nested) -> Iterator[ValidationProblem]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py index 8da220e878..b6f63bcf8f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py @@ -4,25 +4,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Annotated, Final, Literal +from typing import Final -from annotated_types import Interval - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) - -UINT16_DATA_TYPE_NAME: Final = "uint16" -"""The `data_type` value for the `uint16` type.""" - -Uint16DataTypeName = Literal["uint16"] -"""Literal type of the `data_type` field for `uint16`.""" - -Uint16FillValue = Annotated[int, Interval(ge=0, le=2**16 - 1)] -"""Permitted JSON shape of the `fill_value` field for `uint16`: a JSON integer in [0, 65535].""" - +from zarr_metadata.v3._types.data_type import ( + UINT16_DATA_TYPE_NAME, + Uint16DataTypeName, + Uint16FillValue, +) UINT16_DATA_TYPE: Final = DataTypeDefinition( name=UINT16_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py index aba3a4d9d4..b2e8511761 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py @@ -4,25 +4,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Annotated, Final, Literal +from typing import Final -from annotated_types import Interval - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) - -UINT32_DATA_TYPE_NAME: Final = "uint32" -"""The `data_type` value for the `uint32` type.""" - -Uint32DataTypeName = Literal["uint32"] -"""Literal type of the `data_type` field for `uint32`.""" - -Uint32FillValue = Annotated[int, Interval(ge=0, le=2**32 - 1)] -"""Permitted JSON shape of the `fill_value` field for `uint32`: a JSON integer in [0, 2**32 - 1].""" - +from zarr_metadata.v3._types.data_type import ( + UINT32_DATA_TYPE_NAME, + Uint32DataTypeName, + Uint32FillValue, +) UINT32_DATA_TYPE: Final = DataTypeDefinition( name=UINT32_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py index 8dc2140a21..20ba79c979 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py @@ -4,25 +4,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Annotated, Final, Literal +from typing import Final -from annotated_types import Interval - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) - -UINT64_DATA_TYPE_NAME: Final = "uint64" -"""The `data_type` value for the `uint64` type.""" - -Uint64DataTypeName = Literal["uint64"] -"""Literal type of the `data_type` field for `uint64`.""" - -Uint64FillValue = Annotated[int, Interval(ge=0, le=2**64 - 1)] -"""Permitted JSON shape of the `fill_value` field for `uint64`: a JSON integer in [0, 2**64 - 1].""" - +from zarr_metadata.v3._types.data_type import ( + UINT64_DATA_TYPE_NAME, + Uint64DataTypeName, + Uint64FillValue, +) UINT64_DATA_TYPE: Final = DataTypeDefinition( name=UINT64_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py index 74cf0a9a4a..2b8f5d8b10 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py @@ -4,25 +4,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Annotated, Final, Literal +from typing import Final -from annotated_types import Interval - -from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata._common import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, single_byte, ) - -UINT8_DATA_TYPE_NAME: Final = "uint8" -"""The `data_type` value for the `uint8` type.""" - -Uint8DataTypeName = Literal["uint8"] -"""Literal type of the `data_type` field for `uint8`.""" - -Uint8FillValue = Annotated[int, Interval(ge=0, le=2**8 - 1)] -"""Permitted JSON shape of the `fill_value` field for `uint8`: a JSON integer in [0, 255].""" - +from zarr_metadata.v3._types.data_type import ( + UINT8_DATA_TYPE_NAME, + Uint8DataTypeName, + Uint8FillValue, +) UINT8_DATA_TYPE: Final = DataTypeDefinition( name=UINT8_DATA_TYPE_NAME, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py index 11c7da2e69..0882cdfa9e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py @@ -10,22 +10,18 @@ """ import base64 -from typing import Final, Literal +from typing import Final -from zarr_metadata._numpy import Base64Bytes -from zarr_metadata.definition._definition import EmptyConfiguration, Nested +from zarr_metadata._common import EmptyConfiguration +from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import DataTypeDefinition, variable_length +from zarr_metadata.v3._types.data_type import ( + VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME, + VariableLengthBytesDataTypeName, + VariableLengthBytesFillValue, +) from zarr_metadata.v3.data_type._byte import base64_fill_value_rules, base64_spelling -VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME: Final = "variable_length_bytes" -"""The `data_type` value for the `variable_length_bytes` type.""" - -VariableLengthBytesDataTypeName = Literal["variable_length_bytes"] -"""Literal type of the `data_type` field for `variable_length_bytes`.""" - -VariableLengthBytesFillValue = Base64Bytes -"""Permitted JSON shape of the `fill_value` field for `variable_length_bytes`: a `Base64Bytes` string, which is what zarr-python writes and reads.""" - def _fill_value_canonical( configuration: EmptyConfiguration, nested: Nested, value: VariableLengthBytesFillValue diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py index ae89c950b8..05fc869b34 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py @@ -14,48 +14,20 @@ import base64 from collections.abc import Iterator -from typing import Final, Literal, NotRequired - -from typing_extensions import ReadOnly, TypedDict +from typing import Final from zarr_metadata._json import ValidationProblem -from zarr_metadata._numpy import Base64Bytes from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import DataTypeDefinition, single_byte +from zarr_metadata.v3._types.data_type import ( + ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME, + ZarrPythonRawBytes, + ZarrPythonRawBytesConfiguration, + ZarrPythonRawBytesDataTypeName, + ZarrPythonRawBytesFillValue, +) from zarr_metadata.v3.data_type._byte import base64_decoded, base64_problems, base64_spelling -ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME: Final = "raw_bytes" -"""The `name` field value of zarr-python's `raw_bytes` data type.""" - -ZarrPythonRawBytesDataTypeName = Literal["raw_bytes"] -"""Literal type of the `name` field of zarr-python's `raw_bytes` data type.""" - - -class ZarrPythonRawBytesConfiguration(TypedDict, closed=True): - """Configuration for zarr-python's `raw_bytes` data type.""" - - length_bytes: ReadOnly[int] - """The size in bytes of every value, at least 1.""" - - -class ZarrPythonRawBytes(TypedDict, closed=True): - """zarr-python's `raw_bytes` data type metadata.""" - - name: ZarrPythonRawBytesDataTypeName - """The data type's name, `"raw_bytes"`.""" - configuration: ZarrPythonRawBytesConfiguration - """The data type's parameters.""" - must_understand: NotRequired[bool] - """Whether a reader that does not recognize this data type must refuse the document. - - Absent means `true`. The spec does not permit `false` for a data type - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). - """ - - -ZarrPythonRawBytesFillValue = Base64Bytes -"""Permitted JSON shape of the `fill_value` field for zarr-python's `raw_bytes`: the bytes as a `Base64Bytes` string, which zarr-python writes `length_bytes` long.""" - def _rules( configuration: ZarrPythonRawBytesConfiguration, nested: Nested diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 42b859fbb9..4449f78b57 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -57,13 +57,13 @@ variable_length, ) from zarr_metadata.v3._pipeline import read_pipeline +from zarr_metadata.v3._types.data_type import FloatSpecialFillValue from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC -from zarr_metadata.v3.data_type._float import FloatSpecialFillValue from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE From bcc20c872b12b0964eb3372c11d5089bb3736407 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 12 Nov 2026 14:22:17 +0100 Subject: [PATCH 02/11] test(zarr-metadata): check that types modules import no definition Each types-only module is imported in a fresh interpreter in which the `zarr_metadata`, `zarr_metadata.v2` and `zarr_metadata.v3` packages (whose `__init__`s import every definition) are empty stand-ins. The test asserts that no module holding a `Definition` ran. Every other package `__init__` runs for real, so a types module placed in a package that exports definitions fails. A control test checks that the probe does see a definition module when an extension module is imported. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/tests/test_types_modules.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 packages/zarr-metadata/tests/test_types_modules.py diff --git a/packages/zarr-metadata/tests/test_types_modules.py b/packages/zarr-metadata/tests/test_types_modules.py new file mode 100644 index 0000000000..9ef3f499e2 --- /dev/null +++ b/packages/zarr-metadata/tests/test_types_modules.py @@ -0,0 +1,111 @@ +"""Test that no module that holds types imports a module that holds a definition. + +A definition of one kind can then import the types of another kind at run +time: a data type can name its default codec's object form, and a codec +can name the data types it accepts, without an import cycle. + +Python runs a package's `__init__` before any module in it, and the +`__init__` of `zarr_metadata`, `zarr_metadata.v2` and `zarr_metadata.v3` +imports every definition. So each module is imported in a fresh +interpreter in which those three packages are empty stand-ins. Every other +package on the way runs its `__init__` for real, so a types module placed in +a package that exports definitions, such as `zarr_metadata.v3.codec`, fails. +""" + +import importlib +import json +import pkgutil +import subprocess +import sys +from pathlib import Path + +import pytest + +import zarr_metadata +import zarr_metadata.v2._types +import zarr_metadata.v3._types +from zarr_metadata.definition import Definition + +_STAND_INS = ("zarr_metadata", "zarr_metadata.v2", "zarr_metadata.v3") +"""The packages whose `__init__` imports every definition. The probe replaces each with an empty package.""" + +_DOCUMENT_TYPES_MODULES = ( + "zarr_metadata._common", + "zarr_metadata._numpy", + "zarr_metadata.v2._declaration_json", + "zarr_metadata.v2.array", + "zarr_metadata.v2.attributes", + "zarr_metadata.v2.consolidated", + "zarr_metadata.v2.group", + "zarr_metadata.v3._common", + "zarr_metadata.v3.array", + "zarr_metadata.v3.group", +) +"""The modules outside the `_types` packages that hold types only: the documents' types, and the types they share with the extensions.""" + +TYPES_MODULES = ( + *_DOCUMENT_TYPES_MODULES, + *( + info.name + for package in (zarr_metadata.v2._types, zarr_metadata.v3._types) + for info in pkgutil.iter_modules(package.__path__, f"{package.__name__}.") + ), +) +"""Every module that holds types only: the document modules, and each module of the extensions' `_types` packages.""" + +_PROBE = """ +import importlib, json, sys, types + +for name, path in json.loads(sys.argv[1]).items(): + package = types.ModuleType(name) + package.__path__ = [path] + sys.modules[name] = package +importlib.import_module(sys.argv[2]) +print(json.dumps(sorted( + name for name, module in sys.modules.items() + if name.startswith("zarr_metadata") and getattr(module, "__file__", None) is not None +))) +""" +"""Import one module with the stand-ins in place, and print every `zarr_metadata` module that ran.""" + + +def _holds_definition(module_name: str) -> bool: + """Whether the module holds a definition, or a kind of definition, in its namespace.""" + module = importlib.import_module(module_name) + return any( + isinstance(value, Definition) or (isinstance(value, type) and issubclass(value, Definition)) + for value in vars(module).values() + ) + + +DEFINITION_MODULES = frozenset( + info.name + for info in pkgutil.walk_packages(zarr_metadata.__path__, "zarr_metadata.") + if _holds_definition(info.name) +) | frozenset(name for name in _STAND_INS if _holds_definition(name)) +"""Every module of the package that holds a definition or a kind of definition, including a package that re-exports one.""" + + +def _loaded_by(module_name: str) -> frozenset[str]: + """The `zarr_metadata` modules that run when `module_name` is imported in a fresh interpreter with the stand-ins in place.""" + root = Path(zarr_metadata.__path__[0]) + paths = {name: str(root.joinpath(*name.split(".")[1:])) for name in _STAND_INS} + result = subprocess.run( + [sys.executable, "-c", _PROBE, json.dumps(paths), module_name], + capture_output=True, + text=True, + check=True, + ) + return frozenset(json.loads(result.stdout)) + + +@pytest.mark.parametrize("module_name", TYPES_MODULES) +def test_types_module_loads_no_definition(module_name: str) -> None: + loaded = _loaded_by(module_name) + assert module_name in loaded + assert sorted(loaded & DEFINITION_MODULES) == [] + + +def test_probe_sees_a_definition_module() -> None: + """The probe reports a definition module that an extension module imports, so an empty result above is not a probe that sees nothing.""" + assert "zarr_metadata.v3._definition" in _loaded_by("zarr_metadata.v3.codec.gzip") From c12e4e405355b7515db2c5ed79537a2ed0d48431 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 12 Nov 2026 15:35:35 +0100 Subject: [PATCH 03/11] docs(zarr-metadata): number the types-only modules fragment after #4627 Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/changes/{XXXX.misc.md => 4627.misc.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{XXXX.misc.md => 4627.misc.md} (100%) diff --git a/packages/zarr-metadata/changes/XXXX.misc.md b/packages/zarr-metadata/changes/4627.misc.md similarity index 100% rename from packages/zarr-metadata/changes/XXXX.misc.md rename to packages/zarr-metadata/changes/4627.misc.md From 3348a561d15a4d21ac7f0d03f84898f499934fd9 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 12 Nov 2026 17:14:24 +0100 Subject: [PATCH 04/11] refactor(zarr-metadata)!: define each kind of v3 codec with its own class A codec was a `CodecDefinition(kind=...)`, and a table refused the members a codec of a kind is never asked: the chunk rules, transition and pipelines of a bytes -> bytes codec, and the transition of an array -> bytes codec. Each kind is now a class: `ArrayArrayCodecDefinition`, `ArrayBytesCodecDefinition` and `BytesBytesCodecDefinition`. `ArrayCodecDefinition`, the base of the first two, holds `chunk_rules` and `pipelines`, and only an array -> array codec has `transition`, so a bytes -> bytes codec has no member that would be about a chunk. `CodecDefinition` is their abstract base, whose `kind` property each class gives, and it stays the kind that a scope and `resolve` name. The pipeline and the strategies dispatch on the class instead of comparing kind strings, so the type checker sees which codecs have which members. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/changes/XXXX.feature.md | 9 ++ packages/zarr-metadata/docs/extensions.md | 15 +- .../zarr-metadata/docs/parsing-validation.md | 6 +- .../src/zarr_metadata/definition/__init__.py | 7 +- .../src/zarr_metadata/testing/_codec.py | 9 +- .../src/zarr_metadata/testing/_draw.py | 39 +++-- .../src/zarr_metadata/v3/_definition.py | 153 +++++++++++------- .../src/zarr_metadata/v3/_pipeline.py | 13 +- .../src/zarr_metadata/v3/codec/blosc.py | 5 +- .../src/zarr_metadata/v3/codec/bytes.py | 5 +- .../src/zarr_metadata/v3/codec/cast_value.py | 5 +- .../src/zarr_metadata/v3/codec/crc32c.py | 6 +- .../src/zarr_metadata/v3/codec/gzip.py | 5 +- .../zarr_metadata/v3/codec/scale_offset.py | 5 +- .../v3/codec/sharding_indexed.py | 5 +- .../src/zarr_metadata/v3/codec/transpose.py | 5 +- .../src/zarr_metadata/v3/codec/vlen_bytes.py | 5 +- .../src/zarr_metadata/v3/codec/vlen_utf8.py | 5 +- .../src/zarr_metadata/v3/codec/zstd.py | 5 +- .../src/zarr_metadata/v3/definition.py | 19 ++- .../zarr-metadata/tests/model/test_array.py | 4 +- .../tests/model/test_consolidating.py | 12 +- .../tests/model/test_extension_points.py | 5 +- .../zarr-metadata/tests/model/test_group.py | 5 +- .../zarr-metadata/tests/model/test_pair.py | 10 +- .../tests/testing/test_strategies.py | 19 ++- .../tests/v3/test_definitions.py | 92 +++++------ packages/zarr-metadata/tests/v3/test_kinds.py | 5 +- .../zarr-metadata/tests/v3/test_pipelines.py | 46 +++--- packages/zarr-metadata/tests/v3/test_scope.py | 13 +- 30 files changed, 286 insertions(+), 251 deletions(-) create mode 100644 packages/zarr-metadata/changes/XXXX.feature.md diff --git a/packages/zarr-metadata/changes/XXXX.feature.md b/packages/zarr-metadata/changes/XXXX.feature.md new file mode 100644 index 0000000000..cb9e5918d4 --- /dev/null +++ b/packages/zarr-metadata/changes/XXXX.feature.md @@ -0,0 +1,9 @@ +**Breaking:** a v3 codec is defined with the class of its kind, +`ArrayArrayCodecDefinition`, `ArrayBytesCodecDefinition` or +`BytesBytesCodecDefinition`, instead of `CodecDefinition(kind=...)`. The kind +is now the class's `kind` property, and `CodecDefinition` is their abstract +base, which a scope and `resolve` still use to name the kind. A codec that +receives an array, an `ArrayCodecDefinition`, has `chunk_rules` and +`pipelines`, an array -> array codec also has `transition`, and a bytes -> +bytes codec, which receives bytes, has none of them, so building one with any +of them is a `TypeError` from the class itself. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index 5604ce5471..91305dfd96 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -33,7 +33,7 @@ from typing_extensions import TypedDict from zarr_metadata import ValidationProblem from zarr_metadata.definition import Nested -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, BytesBytesCodecDefinition, ZarrV3Context class AcmeLz4Configuration(TypedDict, closed=True): @@ -51,10 +51,9 @@ def acme_lz4_rules( ) -ACME_LZ4 = CodecDefinition( +ACME_LZ4 = BytesBytesCodecDefinition( name="acme.lz4", configuration=AcmeLz4Configuration, - kind="bytes_bytes", size="dynamic", rules=acme_lz4_rules, ) @@ -113,10 +112,12 @@ Other kinds declare more: size. - A chunk grid declares which shapes it fits (`shape_rules`) and its chunk lengths (`chunk_lengths`). -- A codec declares: - - what it checks about the chunk it receives (`chunk_rules`), - - what it hands on (`transition`), - - any pipelines it contains (`pipelines`). +- A codec is defined by the class of its kind, `ArrayArrayCodecDefinition`, + `ArrayBytesCodecDefinition` or `BytesBytesCodecDefinition`. A codec that + receives an array declares what it checks about the chunk it receives + (`chunk_rules`) and any pipelines it contains (`pipelines`), and an + array -> array codec also declares what it hands on (`transition`). A + bytes -> bytes codec receives bytes, so it declares none of these. The [`zarr_metadata.v3.definition`](api/v3/definition.md) docstring describes each of these. diff --git a/packages/zarr-metadata/docs/parsing-validation.md b/packages/zarr-metadata/docs/parsing-validation.md index abdea3c790..1cc0ff2a2f 100644 --- a/packages/zarr-metadata/docs/parsing-validation.md +++ b/packages/zarr-metadata/docs/parsing-validation.md @@ -258,12 +258,12 @@ To make an unknown data type or codec known, add its definition to the context's ```python test="true" session="parsing-validation" from zarr_metadata.definition import EmptyConfiguration -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, BytesBytesCodecDefinition, ZarrV3Context # The definition of the codec that was unknown: here, a codec named # `acme.codec` that takes no configuration. -MY_CODEC = CodecDefinition( - name="acme.codec", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" +MY_CODEC = BytesBytesCodecDefinition( + name="acme.codec", configuration=EmptyConfiguration, size="dynamic" ) context = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[MY_CODEC])) reading = read_array_metadata_v3(raw, context=context) diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 849023c493..4dc31e9162 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -74,7 +74,7 @@ resolved, problems = resolve({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) - resolved # RefusedDeclaration(..., definition=CodecDefinition(name='gzip'), ...) + resolved # RefusedDeclaration(..., definition=BytesBytesCodecDefinition(name='gzip'), ...) problems[0].loc # ('configuration', 'level') problems[0].input # 12 dict(problems[0].ctx) # {'ge': 0, 'le': 9} @@ -125,7 +125,7 @@ from zarr_metadata import ValidationProblem from zarr_metadata.definition import Nested -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, BytesBytesCodecDefinition, ZarrV3Context class AcmeLz4Configuration(TypedDict, closed=True): @@ -143,10 +143,9 @@ def acme_lz4_rules( ) -ACME_LZ4 = CodecDefinition( +ACME_LZ4 = BytesBytesCodecDefinition( name="acme.lz4", configuration=AcmeLz4Configuration, - kind="bytes_bytes", size="dynamic", rules=acme_lz4_rules, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index 2b26c320dd..36cbd5b1d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -81,7 +81,12 @@ from zarr_metadata._common import JSONValue from zarr_metadata.definition._definition import ResolvedDeclaration from zarr_metadata.testing._strategy import Configurations, Strategies - from zarr_metadata.v3._definition import CodecDefinition, CodecSize, Lengths + from zarr_metadata.v3._definition import ( + ArrayBytesCodecDefinition, + CodecDefinition, + CodecSize, + Lengths, + ) _BLOSC_MAX_TYPESIZE: Final = 255 """The widest element c-blosc takes: `BLOSC_MAX_TYPESIZE`.""" @@ -235,7 +240,7 @@ def _zstd(draw: st.DrawFn) -> ZstdCodecConfiguration: return configuration -def _vlen(definition: CodecDefinition[Any]) -> CodecStrategy: +def _vlen(definition: ArrayBytesCodecDefinition[Any]) -> CodecStrategy: """A codec with nothing to configure that takes only some data types: none for a chunk of another, which its own chunk rules say.""" return CodecStrategy( configurations=lambda chunk, _strategies: ( diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py index 6c8c2094d2..f2c0ec3c0b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py @@ -12,7 +12,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, TypeVar, cast from hypothesis import reject from hypothesis import strategies as st @@ -21,7 +21,15 @@ from zarr_metadata.testing._strategy import NOTHING_KNOWN from zarr_metadata.v3._context import ZarrV3Context from zarr_metadata.v3._core import CORE_CONTEXT -from zarr_metadata.v3._definition import CodecDefinition, DataTypeDefinition, no_pipelines +from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, + ArrayBytesCodecDefinition, + ArrayCodecDefinition, + BytesBytesCodecDefinition, + CodecDefinition, + DataTypeDefinition, + no_pipelines, +) if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Sequence @@ -39,7 +47,10 @@ ) from zarr_metadata.v3 import ZarrV3DeclarationJSON from zarr_metadata.v3._context import ZarrV3ContextSlots - from zarr_metadata.v3._definition import Chunk, CodecKind, CodecSize + from zarr_metadata.v3._definition import Chunk, CodecSize + +K = TypeVar("K", bound="CodecDefinition[Any]") +"""A kind of codec: the class of its definitions.""" _MAX_ARRAY_ARRAY: Final = 2 """The most array -> array codecs a pipeline is drawn with.""" @@ -221,20 +232,20 @@ def fill_values_of( def _taking( chunk: Chunk, - kind: CodecKind, + kind: type[K], sizes: frozenset[CodecSize], strategies: Strategies, *, holding: bool = True, -) -> list[tuple[CodecDefinition[Any], Configurations]]: +) -> list[tuple[K, Configurations]]: """Each codec of `kind` and one of `sizes` that `strategies` draw for, that takes `chunk`, with its configurations. Without `holding`, a codec that holds pipelines of its own is left out: what one of those takes depends on what its pipelines do. """ - taking: list[tuple[CodecDefinition[Any], Configurations]] = [] + taking: list[tuple[K, Configurations]] = [] for definition in sorted(scope_of(strategies).codecs.values(), key=lambda d: d.name): - if definition.kind != kind or definition.size not in sizes: + if not isinstance(definition, kind) or definition.size not in sizes: continue if not holding and _holds_pipelines(definition): continue @@ -246,7 +257,7 @@ def _taking( def _holds_pipelines(definition: CodecDefinition[Any]) -> bool: - return definition.pipelines is not no_pipelines + return isinstance(definition, ArrayCodecDefinition) and definition.pipelines is not no_pipelines def pipelines_for( @@ -258,15 +269,15 @@ def pipelines_for( that holds pipelines of its own takes a chunk only if what it holds does, so it settles nothing. """ - if len(_taking(chunk, "array_bytes", sizes, strategies, holding=False)) == 0: + if len(_taking(chunk, ArrayBytesCodecDefinition, sizes, strategies, holding=False)) == 0: return None return _walk(chunk, strategies, sizes) @st.composite def _codec( - draw: st.DrawFn, options: Sequence[tuple[CodecDefinition[Any], Configurations]] -) -> tuple[CodecDefinition[Any], ZarrV3DeclarationJSON]: + draw: st.DrawFn, options: Sequence[tuple[K, Configurations]] +) -> tuple[K, ZarrV3DeclarationJSON]: definition, configurations = draw(st.sampled_from(options)) return definition, draw(any_spelling(definition, configurations)) @@ -285,7 +296,7 @@ def _walk( codecs: list[ZarrV3DeclarationJSON] = [] handed = chunk for _ in range(draw(st.integers(0, _MAX_ARRAY_ARRAY))): - options = _taking(handed, "array_array", sizes, strategies) + options = _taking(handed, ArrayArrayCodecDefinition, sizes, strategies) if len(options) == 0: break definition, declaration = draw(_codec(options)) @@ -293,11 +304,11 @@ def _walk( assert isinstance(codec, AcceptedDeclaration) handed = definition.transition(codec.configuration, codec.nested, handed) codecs.append(declaration) - options = _taking(handed, "array_bytes", sizes, strategies) + options = _taking(handed, ArrayBytesCodecDefinition, sizes, strategies) if len(options) == 0: reject() # what the array -> array codecs handed on, no array -> bytes codec takes codecs.append(draw(_codec(options))[1]) - options = _taking(NOTHING_KNOWN, "bytes_bytes", sizes, strategies) + options = _taking(NOTHING_KNOWN, BytesBytesCodecDefinition, sizes, strategies) if len(options) != 0: count = draw(st.integers(0, _MAX_BYTES_BYTES)) codecs.extend(draw(_codec(options))[1] for _ in range(count)) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index a8171b7e61..6100aa7df7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -11,8 +11,8 @@ from __future__ import annotations -import dataclasses import re +from abc import ABC, abstractmethod from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass from typing import ( @@ -300,48 +300,18 @@ class ChunkKeyEncodingDefinition(Definition[C], kind=True, format=3): """ -_UNASKED: Final[Mapping[CodecKind, tuple[str, ...]]] = { - "array_array": (), - "array_bytes": ("transition",), - "bytes_bytes": ("chunk_rules", "transition", "pipelines"), -} -"""The functions that are never called for a codec of each kind. A bytes -> bytes codec receives bytes, and only an array -> array codec passes on a chunk.""" - - @dataclass(frozen=True, kw_only=True, slots=True, repr=False) -class CodecDefinition(Definition[C], kind=True, format=3): +class CodecDefinition(Definition[C], ABC, kind=True, format=3): """A codec: what it converts its input to, and whether the size of its output is static. - A codec that receives an array (array -> array, array -> bytes) - declares the spec violations for a given input `Chunk` with - `chunk_rules`. It receives the configuration, the nested declarations as the - scope read them, and the chunk, and it locates its problems in the - configuration. For example, a `bytes` codec without `endian` that - receives a multi-byte data type is a problem. An array -> array codec - also declares its output with `transition`: given its input chunk, it - returns the chunk the next codec receives. For example, `transpose` - permutes the axes, and `cast_value` changes the data type. These two - functions correspond to the spec: a codec computes its output from the - shape and data type of its input, and "If the - decoded_representation_type is not supported, this algorithm must - fail with an error" - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L987-L994). - The transition is called for every input chunk, whatever the chunk - rules found, so it must return only what is true in either case. For - example, a `transpose` whose `order` has the wrong number of axes - passes on unknown lengths. A codec that does not declare its output - passes the next codec a chunk with nothing known about it. - - A codec that contains its own pipelines declares what each one - receives with `pipelines`. Given the codec's input chunk, it returns - the chunk that the first codec of each pipeline receives, keyed by the - configuration member that holds the pipeline. For example, a shard's - inner codecs receive its inner chunks, and its index codecs receive - the shard index. Like the transition, it is called whatever the chunk - rules found, and must return only what is true in either case. A - function that is never called for a codec of its kind is rejected, - such as the chunk rules of a bytes -> bytes codec, which receives - bytes. + A codec is one of three kinds, each its own class: + `ArrayArrayCodecDefinition`, `ArrayBytesCodecDefinition` and + `BytesBytesCodecDefinition`. The kind fixes where the codec may sit in + a pipeline, and what it is asked: a codec that receives an array, an + `ArrayCodecDefinition`, is asked about the chunk it receives, and a + bytes -> bytes codec, which receives bytes, is asked nothing about + them. This class holds what every codec declares, and is not built + itself. """ label: ClassVar[str] = "codec" @@ -356,39 +326,22 @@ class CodecDefinition(Definition[C], kind=True, format=3): codecs of static size. """ - kind: CodecKind - """What the codec does to what it is handed: array -> array, array -> bytes or bytes -> bytes. - - It fixes where the codec may sit in a pipeline, and which of `chunk_rules`, - `transition` and `pipelines` it may be given: building a definition with - one its kind is never asked raises `TypeError`. - """ size: CodecSize """Whether the size of what the codec gives out is fixed by the size of what it is handed. Where a member takes only codecs of static size, a codec of dynamic size is a problem. """ - chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules - """The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec.""" - transition: Callable[[C, Nested, Chunk], Chunk] = unknown_chunk - """The chunk the next codec receives, given the chunk this array -> array codec receives.""" - pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines - """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" + + @property + @abstractmethod + def kind(self) -> CodecKind: + """What the codec does to what it is handed: array -> array, array -> bytes or bytes -> bytes. It fixes where the codec may sit in a pipeline.""" def _refusal(self) -> str | None: - kind: object = self.kind - if kind not in get_args(CodecKind): - return f"{self.name!r}: kind is one of {get_args(CodecKind)!r}, got {kind!r}" size: object = self.size if size not in get_args(CodecSize): return f"{self.name!r}: size is one of {get_args(CodecSize)!r}, got {size!r}" - defaults = { - attribute.name: attribute.default for attribute in dataclasses.fields(CodecDefinition) - } - for attribute in _UNASKED[self.kind]: - if getattr(self, attribute) is not defaults[attribute]: - return f"{self.name!r}: {attribute}, which no codec of kind {kind!r} is asked" return None @classmethod @@ -408,6 +361,78 @@ def _alias_problems( ) +@dataclass(frozen=True, kw_only=True, slots=True, repr=False) +class ArrayCodecDefinition(CodecDefinition[C]): + """A codec that receives an array: an array -> array or an array -> bytes codec. + + It declares the spec violations for a given input `Chunk` with + `chunk_rules`. They receive the configuration, the nested declarations + as the scope read them, and the chunk, and they locate their problems + in the configuration. For example, a `bytes` codec without `endian` + that receives a multi-byte data type is a problem. This corresponds to + the spec: a codec computes its output from the shape and data type of + its input, and "If the decoded_representation_type is not supported, + this algorithm must fail with an error" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L987-L994). + + A codec that contains its own pipelines declares what each one + receives with `pipelines`. Given the codec's input chunk, it returns + the chunk that the first codec of each pipeline receives, keyed by the + configuration member that holds the pipeline. For example, a shard's + inner codecs receive its inner chunks, and its index codecs receive + the shard index. It is called whatever the chunk rules found, and must + return only what is true in either case. + """ + + chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules + """The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec.""" + pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines + """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" + + +@dataclass(frozen=True, kw_only=True, slots=True, repr=False) +class ArrayArrayCodecDefinition(ArrayCodecDefinition[C]): + """An array -> array codec, such as `transpose`. + + It declares its output with `transition`: given its input chunk, it + returns the chunk the next codec receives. For example, `transpose` + permutes the axes, and `cast_value` changes the data type. The + transition is called for every input chunk, whatever the chunk rules + found, so it must return only what is true in either case. For + example, a `transpose` whose `order` has the wrong number of axes + passes on unknown lengths. A codec that does not declare its output + passes the next codec a chunk with nothing known about it. + """ + + transition: Callable[[C, Nested, Chunk], Chunk] = unknown_chunk + """The chunk the next codec receives, given the chunk this codec receives.""" + + @property + def kind(self) -> CodecKind: + """`"array_array"`: the codec sits before the array -> bytes codec, and hands an array on.""" + return "array_array" + + +@dataclass(frozen=True, kw_only=True, slots=True, repr=False) +class ArrayBytesCodecDefinition(ArrayCodecDefinition[C]): + """An array -> bytes codec, such as `bytes`: the one codec of a pipeline that turns the array into bytes.""" + + @property + def kind(self) -> CodecKind: + """`"array_bytes"`: the codec turns the array into bytes, exactly once in a pipeline.""" + return "array_bytes" + + +@dataclass(frozen=True, kw_only=True, slots=True, repr=False) +class BytesBytesCodecDefinition(CodecDefinition[C]): + """A bytes -> bytes codec, such as `gzip`. It receives bytes, which have no shape or data type, so it is asked nothing about them.""" + + @property + def kind(self) -> CodecKind: + """`"bytes_bytes"`: the codec sits after the array -> bytes codec.""" + return "bytes_bytes" + + @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class StorageTransformerDefinition(Definition[C], kind=True, format=3): """A storage transformer.""" @@ -796,7 +821,11 @@ def chunk_grid_lengths( "RAW_BYTES_NAME", "RAW_BYTES_NAME_PATTERN", "STATIC_SIZE_ALIASES", + "ArrayArrayCodecDefinition", + "ArrayBytesCodecDefinition", + "ArrayCodecDefinition", "ArrayExtraFieldDefinition", + "BytesBytesCodecDefinition", "Chunk", "ChunkGridDefinition", "ChunkKeyEncodingDefinition", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index a46883d439..592bb18a5f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -43,6 +43,8 @@ ruled, ) from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, + ArrayCodecDefinition, Chunk, CodecDefinition, CodecKind, @@ -131,7 +133,8 @@ def read_pipeline( stages.append(Stage(codec, handed)) handed = None continue - if definition.kind == "bytes_bytes": + if not isinstance(definition, ArrayCodecDefinition): + # A bytes -> bytes codec receives bytes, which are not a chunk. stages.append(Stage(codec, None)) handed = None continue @@ -145,7 +148,7 @@ def read_pipeline( inner, found = _inner_pipelines(definition, configuration, nested, incoming, at) problems.extend(found) stages.append(Stage(codec, incoming, inner)) - if definition.kind == "array_bytes": + if not isinstance(definition, ArrayArrayCodecDefinition): handed = None elif not isinstance(codec, AcceptedDeclaration): handed = Chunk() @@ -202,7 +205,7 @@ def _order_problems( def _chunk_problems( - definition: CodecDefinition[Any], + definition: ArrayCodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, chunk: Chunk, @@ -235,7 +238,7 @@ def _codec_problems( def _inner_pipelines( - definition: CodecDefinition[Any], + definition: ArrayCodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, chunk: Chunk, @@ -299,7 +302,7 @@ def _held( def _handed_on( - definition: CodecDefinition[Any], + definition: ArrayArrayCodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, chunk: Chunk, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 5aac8b054d..0ed7cc40a3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -9,7 +9,7 @@ from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested -from zarr_metadata.v3._definition import CodecDefinition +from zarr_metadata.v3._definition import BytesBytesCodecDefinition from zarr_metadata.v3._types.codec import ( BLOSC_CNAME, BLOSC_CODEC_NAME, @@ -62,10 +62,9 @@ def _canonical(configuration: BloscCodecConfiguration) -> BloscCodecConfiguratio return simpler -BLOSC_CODEC: Final = CodecDefinition( +BLOSC_CODEC: Final = BytesBytesCodecDefinition( name=BLOSC_CODEC_NAME, configuration=BloscCodecConfiguration, - kind="bytes_bytes", size="dynamic", rules=_rules, canonical=_canonical, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 652c3d9f5f..5b08cf3b0a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -10,8 +10,8 @@ from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import ( + ArrayBytesCodecDefinition, Chunk, - CodecDefinition, storage_of, ) from zarr_metadata.v3._types.codec import ( @@ -64,10 +64,9 @@ def _chunk_rules( ) -BYTES_CODEC: Final = CodecDefinition( +BYTES_CODEC: Final = ArrayBytesCodecDefinition( name=BYTES_CODEC_NAME, configuration=BytesCodecConfiguration, - kind="array_bytes", size="static", chunk_rules=_chunk_rules, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index c00af8b8b1..2520f9e2eb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -15,8 +15,8 @@ fill_value_problems, ) from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, Chunk, - CodecDefinition, ) from zarr_metadata.v3._types.codec import ( CAST_OUT_OF_RANGE_MODE, @@ -129,10 +129,9 @@ def _transition(configuration: CastValueCodecConfiguration, nested: Nested, chun return Chunk(chunk.lengths, nested.get(("data_type",))) -CAST_VALUE_CODEC: Final = CodecDefinition( +CAST_VALUE_CODEC: Final = ArrayArrayCodecDefinition( name=CAST_VALUE_CODEC_NAME, configuration=CastValueCodecConfiguration, - kind="array_array", size="static", rules=_rules, chunk_rules=_chunk_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 0c94867d5a..bf1f2fe184 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -10,7 +10,7 @@ from typing import Final from zarr_metadata._common import EmptyConfiguration -from zarr_metadata.v3._definition import CodecDefinition +from zarr_metadata.v3._definition import BytesBytesCodecDefinition from zarr_metadata.v3._types.codec import ( CRC32C_CODEC_NAME, Crc32cCodecMetadata, @@ -18,8 +18,8 @@ Crc32cCodecObject, ) -CRC32C_CODEC: Final = CodecDefinition( - name=CRC32C_CODEC_NAME, configuration=EmptyConfiguration, kind="bytes_bytes", size="static" +CRC32C_CODEC: Final = BytesBytesCodecDefinition( + name=CRC32C_CODEC_NAME, configuration=EmptyConfiguration, size="static" ) """The `crc32c` codec. A checksum has no configuration.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 126e778797..1778e090cb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -6,7 +6,7 @@ from typing import Final -from zarr_metadata.v3._definition import CodecDefinition +from zarr_metadata.v3._definition import BytesBytesCodecDefinition from zarr_metadata.v3._types.codec import ( GZIP_CODEC_NAME, GzipCodecConfiguration, @@ -15,10 +15,9 @@ GzipCodecObject, ) -GZIP_CODEC: Final = CodecDefinition( +GZIP_CODEC: Final = BytesBytesCodecDefinition( name=GZIP_CODEC_NAME, configuration=GzipCodecConfiguration, - kind="bytes_bytes", size="dynamic", ) """The `gzip` codec.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 473eeb4c1f..6b9782e345 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -14,8 +14,8 @@ fill_value_problems, ) from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, Chunk, - CodecDefinition, ) from zarr_metadata.v3._types.codec import ( SCALE_OFFSET_CODEC_NAME, @@ -86,10 +86,9 @@ def _transition( return chunk -SCALE_OFFSET_CODEC: Final = CodecDefinition( +SCALE_OFFSET_CODEC: Final = ArrayArrayCodecDefinition( name=SCALE_OFFSET_CODEC_NAME, configuration=ScaleOffsetCodecConfiguration, - kind="array_array", size="static", rules=_rules, chunk_rules=_chunk_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 5b4b3556e8..5a4d0f5983 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -13,8 +13,8 @@ Nested, ) from zarr_metadata.v3._definition import ( + ArrayBytesCodecDefinition, Chunk, - CodecDefinition, Lengths, ) from zarr_metadata.v3._types.codec import ( @@ -134,11 +134,10 @@ def _canonical( return simpler -SHARDING_INDEXED_CODEC: Final = CodecDefinition( +SHARDING_INDEXED_CODEC: Final = ArrayBytesCodecDefinition( name=SHARDING_INDEXED_CODEC_NAME, configuration=ShardingIndexedCodecConfiguration, canonical=_canonical, - kind="array_bytes", size="dynamic", chunk_rules=_chunk_rules, pipelines=_pipelines, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index b4e34b2ab6..7d0adb9446 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -10,8 +10,8 @@ from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, Chunk, - CodecDefinition, ) from zarr_metadata.v3._types.codec import ( TRANSPOSE_CODEC_NAME, @@ -74,10 +74,9 @@ def _transition(configuration: TransposeCodecConfiguration, nested: Nested, chun return Chunk(tuple(lengths[axis] for axis in order), chunk.data_type) -TRANSPOSE_CODEC: Final = CodecDefinition( +TRANSPOSE_CODEC: Final = ArrayArrayCodecDefinition( name=TRANSPOSE_CODEC_NAME, configuration=TransposeCodecConfiguration, - kind="array_array", size="static", rules=_rules, chunk_rules=_chunk_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py index 8b2f42da68..a8ebdd230f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py @@ -14,7 +14,7 @@ from zarr_metadata._common import EmptyConfiguration from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested -from zarr_metadata.v3._definition import Chunk, CodecDefinition +from zarr_metadata.v3._definition import ArrayBytesCodecDefinition, Chunk from zarr_metadata.v3._types.codec import ( VLEN_BYTES_CODEC_NAME, VlenBytesCodecMetadata, @@ -42,10 +42,9 @@ def _chunk_rules( yield from vlen_problems(chunk, _COMPATIBLE) -VLEN_BYTES_CODEC: Final = CodecDefinition( +VLEN_BYTES_CODEC: Final = ArrayBytesCodecDefinition( name=VLEN_BYTES_CODEC_NAME, configuration=EmptyConfiguration, - kind="array_bytes", size="dynamic", chunk_rules=_chunk_rules, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py index a698288646..cb58373fbb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py @@ -14,7 +14,7 @@ from zarr_metadata._common import EmptyConfiguration from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested -from zarr_metadata.v3._definition import Chunk, CodecDefinition +from zarr_metadata.v3._definition import ArrayBytesCodecDefinition, Chunk from zarr_metadata.v3._types.codec import ( VLEN_UTF8_CODEC_NAME, VlenUtf8CodecMetadata, @@ -34,10 +34,9 @@ def _chunk_rules( yield from vlen_problems(chunk, _COMPATIBLE) -VLEN_UTF8_CODEC: Final = CodecDefinition( +VLEN_UTF8_CODEC: Final = ArrayBytesCodecDefinition( name=VLEN_UTF8_CODEC_NAME, configuration=EmptyConfiguration, - kind="array_bytes", size="dynamic", chunk_rules=_chunk_rules, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index a4153ca6f8..837f280921 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -8,7 +8,7 @@ from typing import Final -from zarr_metadata.v3._definition import CodecDefinition +from zarr_metadata.v3._definition import BytesBytesCodecDefinition from zarr_metadata.v3._types.codec import ( ZSTD_CODEC_NAME, ZSTD_MAX_LEVEL, @@ -34,11 +34,10 @@ def _canonical(configuration: ZstdCodecConfiguration) -> ZstdCodecConfiguration: return simpler -ZSTD_CODEC: Final = CodecDefinition( +ZSTD_CODEC: Final = BytesBytesCodecDefinition( name=ZSTD_CODEC_NAME, configuration=ZstdCodecConfiguration, canonical=_canonical, - kind="bytes_bytes", size="dynamic", ) """The `zstd` codec.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index 4b06e91a22..b3777f190c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -1,9 +1,10 @@ """The kinds of metadata in a Zarr v3 document, and the scopes this package provides for it. The kind of metadata a definition defines is its type: `CodecDefinition` -(with the codec's `kind`, and its `size`: `"static"` if the output size -is determined by the input size, or `"dynamic"` if it depends on the -values), `DataTypeDefinition`, `ChunkGridDefinition`, +(built as one of its kinds, `ArrayArrayCodecDefinition`, +`ArrayBytesCodecDefinition` or `BytesBytesCodecDefinition`, with its +`size`: `"static"` if the output size is determined by the input size, +or `"dynamic"` if it depends on the values), `DataTypeDefinition`, `ChunkGridDefinition`, `ChunkKeyEncodingDefinition`, `StorageTransformerDefinition`, and, for the members of a document that the spec does not define, `ArrayExtraFieldDefinition` and `GroupExtraFieldDefinition`. What a @@ -68,8 +69,8 @@ A codec is checked against its input. The array passes its first codec a `Chunk`: the lengths of the grid's chunks along each of the array's dimensions, and the array's data type declaration, with None for unknown -parts. A codec that receives an array declares the spec violations for -a given chunk with `chunk_rules`, located in its configuration (for +parts. A codec that receives an array, an `ArrayCodecDefinition`, +declares the spec violations for a given chunk with `chunk_rules`, located in its configuration (for example, a `transpose` whose `order` has the wrong number of axes). An array -> array codec declares what it passes to the next codec with `transition`, whatever its chunk rules found. For example, `transpose` @@ -118,7 +119,11 @@ ZARR_PYTHON_CONTEXT, ) from zarr_metadata.v3._definition import ( + ArrayArrayCodecDefinition, + ArrayBytesCodecDefinition, + ArrayCodecDefinition, ArrayExtraFieldDefinition, + BytesBytesCodecDefinition, Chunk, ChunkGridDefinition, ChunkKeyEncodingDefinition, @@ -140,7 +145,11 @@ "CORE_AND_EXTENSIONS_CONTEXT", "CORE_CONTEXT", "ZARR_PYTHON_CONTEXT", + "ArrayArrayCodecDefinition", + "ArrayBytesCodecDefinition", + "ArrayCodecDefinition", "ArrayExtraFieldDefinition", + "BytesBytesCodecDefinition", "Chunk", "ChunkGridDefinition", "ChunkKeyEncodingDefinition", diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 65cccd3dde..7fd62c5442 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -61,6 +61,7 @@ CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, ZARR_PYTHON_CONTEXT, + ArrayBytesCodecDefinition, ArrayExtraFieldDefinition, CodecDefinition, DataTypeDefinition, @@ -388,10 +389,9 @@ def _acme_vlen_text(configuration: EmptyConfiguration, nested: Nested) -> "ZarrV ), ], codecs=[ - CodecDefinition( + ArrayBytesCodecDefinition( name="acme.vlen-text", configuration=EmptyConfiguration, - kind="array_bytes", size="dynamic", ) ], diff --git a/packages/zarr-metadata/tests/model/test_consolidating.py b/packages/zarr-metadata/tests/model/test_consolidating.py index 8e5d5b505f..04e60cc1a4 100644 --- a/packages/zarr-metadata/tests/model/test_consolidating.py +++ b/packages/zarr-metadata/tests/model/test_consolidating.py @@ -25,7 +25,7 @@ from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, - CodecDefinition, + ArrayBytesCodecDefinition, ZarrV3Context, ) @@ -38,9 +38,7 @@ LOOSE_ZSTD = {"name": "zstd", "configuration": {"level": 30}} WITH_ZSTD = {**ARRAY, "codecs": (*ARRAY["codecs"], ZSTD)} WITH_LOOSE_ZSTD = {**ARRAY, "codecs": (*ARRAY["codecs"], LOOSE_ZSTD)} -MY_BYTES = CodecDefinition( - name="bytes", configuration=EmptyConfiguration, kind="array_bytes", size="static" -) +MY_BYTES = ArrayBytesCodecDefinition(name="bytes", configuration=EmptyConfiguration, size="static") PRIVATE = CORE_CONTEXT.copy(update={"codecs": {**CORE_CONTEXT.codecs, "bytes": MY_BYTES}}) INLINE: dict[str, Any] = {"kind": "inline", "must_understand": False} @@ -135,8 +133,8 @@ def test_error_a_model_that_conflicts_with_the_scope_is_refused_at_its_path() -> assert problem.kind == "invalid_value" assert problem.message == ( "expected a document read in the scope of the consolidated metadata, got a model that " - "reads the codec 'bytes' by CodecDefinition(name='bytes') of EmptyConfiguration, which " - "that scope reads by CodecDefinition(name='bytes') of BytesCodecConfiguration" + "reads the codec 'bytes' by ArrayBytesCodecDefinition(name='bytes') of EmptyConfiguration, which " + "that scope reads by ArrayBytesCodecDefinition(name='bytes') of BytesCodecConfiguration" ) @@ -341,7 +339,7 @@ def test_error_a_conflict_between_definitions_alike_says_they_differ() -> None: assert problem.message == ( "expected a document read in the scope of the consolidated metadata, got a model that " "reads the codec 'gzip' by another definition than the one that scope reads it by, " - "CodecDefinition(name='gzip') of GzipCodecConfiguration" + "BytesBytesCodecDefinition(name='gzip') of GzipCodecConfiguration" ) diff --git a/packages/zarr-metadata/tests/model/test_extension_points.py b/packages/zarr-metadata/tests/model/test_extension_points.py index 14b58999a3..e7c9fecc98 100644 --- a/packages/zarr-metadata/tests/model/test_extension_points.py +++ b/packages/zarr-metadata/tests/model/test_extension_points.py @@ -27,7 +27,7 @@ from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, - CodecDefinition, + BytesBytesCodecDefinition, ) if TYPE_CHECKING: @@ -48,10 +48,9 @@ class LenientGzipConfiguration(TypedDict, closed=True): level: int -LENIENT_GZIP = CodecDefinition( +LENIENT_GZIP = BytesBytesCodecDefinition( name="gzip", configuration=LenientGzipConfiguration, - kind="bytes_bytes", size="dynamic", ) """A reader's own gzip definition, which accepts any level. A scope can be extended, and an extension can replace a definition.""" diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index c60d1e3618..a0956e93c0 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -51,6 +51,7 @@ from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, + BytesBytesCodecDefinition, CodecDefinition, ZarrV3Context, ) @@ -642,9 +643,7 @@ def test_error_a_consolidated_envelope_reports_the_members_it_lacks_in_its_order ] -ACME_X = CodecDefinition( - name="acme.x", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" -) +ACME_X = BytesBytesCodecDefinition(name="acme.x", configuration=EmptyConfiguration, size="dynamic") def test_group_replace_keeps_the_documents_it_holds() -> None: diff --git a/packages/zarr-metadata/tests/model/test_pair.py b/packages/zarr-metadata/tests/model/test_pair.py index 5800762e2f..998a45e3f2 100644 --- a/packages/zarr-metadata/tests/model/test_pair.py +++ b/packages/zarr-metadata/tests/model/test_pair.py @@ -36,7 +36,8 @@ from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, - CodecDefinition, + ArrayBytesCodecDefinition, + BytesBytesCodecDefinition, ZarrV3Context, ) @@ -53,9 +54,7 @@ "chunk_key_encoding": {"name": "default"}, "codecs": ["bytes"], } -MY_BYTES = CodecDefinition( - name="bytes", configuration=EmptyConfiguration, kind="array_bytes", size="static" -) +MY_BYTES = ArrayBytesCodecDefinition(name="bytes", configuration=EmptyConfiguration, size="static") """A private `bytes` definition: a different meaning under the core name, with no configuration.""" PRIVATE = CORE_CONTEXT.copy(update={"codecs": {**CORE_CONTEXT.codecs, "bytes": MY_BYTES}}) SPELLED_OUT: dict[str, Any] = { @@ -178,10 +177,9 @@ def test_a_model_round_trips_through_its_document_in_its_scope( def test_error_a_model_whose_scope_does_not_pickle_says_so() -> None: """A scope that holds a definition with a local function cannot be pickled. Pickling the model raises the same error as pickling the `Context`.""" - local = CodecDefinition( + local = BytesBytesCodecDefinition( name="acme.c", configuration=EmptyConfiguration, - kind="bytes_bytes", size="dynamic", rules=lambda configuration, nested: iter(()), ) diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 15800e471a..b947741afe 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -80,6 +80,9 @@ from zarr_metadata.v3.definition import ( CORE_CONTEXT, ZARR_PYTHON_CONTEXT, + ArrayBytesCodecDefinition, + ArrayCodecDefinition, + BytesBytesCodecDefinition, Chunk, ChunkGridDefinition, CodecDefinition, @@ -236,6 +239,8 @@ def _chunks(draw: st.DrawFn) -> Chunk: def _handed_problems(codec: Any, chunk: Chunk) -> list[object]: # noqa: ANN401 """What is wrong with `codec`, a codec field read, handed `chunk`: by its chunk rules, and in each pipeline it holds.""" definition = codec.definition + if not isinstance(definition, ArrayCodecDefinition): + return [] # a bytes -> bytes codec is handed bytes, not the chunk problems: list[object] = list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) handed = definition.pipelines(codec.configuration, codec.nested, chunk) for member, inner in handed.items(): @@ -245,13 +250,15 @@ def _handed_problems(codec: Any, chunk: Chunk) -> list[object]: # noqa: ANN401 def _holds_pipelines(definition: CodecDefinition[Any]) -> bool: - return definition.pipelines is not no_pipelines + return isinstance(definition, ArrayCodecDefinition) and definition.pipelines is not no_pipelines def _encoders_refuse(data: st.DataObject, chunk: Chunk) -> bool: """Whether every array -> bytes codec that holds no pipelines refuses a chunk of `chunk`'s data type.""" of_data_type = Chunk(None, chunk.data_type) - encoders = [d for d in _CODECS if d.kind == "array_bytes" and not _holds_pipelines(d)] + encoders = [ + d for d in _CODECS if isinstance(d, ArrayBytesCodecDefinition) and not _holds_pipelines(d) + ] for encoder in encoders: codec = _read(encoder, data.draw(declarations(encoder))) if list(encoder.chunk_rules(codec.configuration, codec.nested, of_data_type)) == []: @@ -277,6 +284,7 @@ def test_codecs_take_their_chunk( if _holds_pipelines(definition): assert _encoders_refuse(data, chunk) return + assert isinstance(definition, ArrayCodecDefinition) codec = _read(definition, data.draw(declarations(definition))) assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) != [] return @@ -966,8 +974,8 @@ def test_cast_value_draws_only_real_number_data_types(declaration: ZarrV3Declara assert target.definition in _REAL_NUMBERS -_PLAIN_GZIP = CodecDefinition( - name=GZIP_CODEC.name, configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" +_PLAIN_GZIP = BytesBytesCodecDefinition( + name=GZIP_CODEC.name, configuration=EmptyConfiguration, size="dynamic" ) """A codec named as one the defaults hold, with a configuration of its own.""" @@ -1010,10 +1018,9 @@ def test_a_later_definition_replaces_a_default_of_its_name(pipeline: tuple[Any, _PLAIN_STRUCT = DataTypeDefinition(name=STRUCT_DATA_TYPE.name, configuration=EmptyConfiguration) """A data type named as the struct the defaults hold, with a configuration of its own.""" -_PLAIN_SHARD = CodecDefinition( +_PLAIN_SHARD = ArrayBytesCodecDefinition( name=SHARDING_INDEXED_CODEC.name, configuration=EmptyConfiguration, - kind="array_bytes", size="dynamic", ) """A codec named as the shard the defaults hold, with a configuration of its own.""" diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index f261692fb8..d88dcf639e 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -57,6 +57,8 @@ from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, + ArrayArrayCodecDefinition, + BytesBytesCodecDefinition, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, @@ -100,10 +102,9 @@ def acme_stack_rules( ) -ACME_STACK = CodecDefinition( +ACME_STACK = BytesBytesCodecDefinition( name="acme.stack", configuration=AcmeStackConfiguration, - kind="bytes_bytes", size="dynamic", rules=acme_stack_rules, ) @@ -176,35 +177,32 @@ def acme_paired_rules( yield ValidationProblem((), "expected first and second, or neither", "invalid_value") -ACME_LEVEL = CodecDefinition( - name="acme.level", configuration=AcmeLevelConfiguration, kind="bytes_bytes", size="dynamic" +ACME_LEVEL = BytesBytesCodecDefinition( + name="acme.level", configuration=AcmeLevelConfiguration, size="dynamic" ) -ACME_TOTAL = CodecDefinition( - name="acme.total", configuration=AcmeTotalConfiguration, kind="bytes_bytes", size="dynamic" +ACME_TOTAL = BytesBytesCodecDefinition( + name="acme.total", configuration=AcmeTotalConfiguration, size="dynamic" ) -ACME_TREE = CodecDefinition( - name="acme.tree", configuration=AcmeTreeConfiguration, kind="bytes_bytes", size="dynamic" +ACME_TREE = BytesBytesCodecDefinition( + name="acme.tree", configuration=AcmeTreeConfiguration, size="dynamic" ) -ACME_FALLBACK = CodecDefinition( +ACME_FALLBACK = BytesBytesCodecDefinition( name="acme.fallback", configuration=AcmeFallbackConfiguration, - kind="bytes_bytes", size="dynamic", ) -ACME_ROUTES = CodecDefinition( - name="acme.routes", configuration=AcmeRoutesConfiguration, kind="bytes_bytes", size="dynamic" +ACME_ROUTES = BytesBytesCodecDefinition( + name="acme.routes", configuration=AcmeRoutesConfiguration, size="dynamic" ) -ACME_BOUNDED = CodecDefinition( +ACME_BOUNDED = BytesBytesCodecDefinition( name="acme.bounded", configuration=AcmeBoundedConfiguration, - kind="bytes_bytes", size="dynamic", rules=acme_bounded_rules, ) -ACME_PAIRED = CodecDefinition( +ACME_PAIRED = BytesBytesCodecDefinition( name="acme.paired", configuration=AcmePairedConfiguration, - kind="bytes_bytes", size="dynamic", rules=acme_paired_rules, ) @@ -373,8 +371,8 @@ class _HoldsV3Codecs(TypedDict, closed=True): ("build", "named"), [ ( - lambda: CodecDefinition( - name="acme.mix", configuration=_HoldsAV2DataType, kind="array_array", size="static" + lambda: ArrayArrayCodecDefinition( + name="acme.mix", configuration=_HoldsAV2DataType, size="static" ), "ZarrV2DataTypeDeclaration", ), @@ -383,16 +381,15 @@ class _HoldsV3Codecs(TypedDict, closed=True): "ZarrV3CodecDeclaration", ), ( - lambda: CodecDefinition( - name="acme.mix", configuration=_HoldsV2DataTypes, kind="array_array", size="static" + lambda: ArrayArrayCodecDefinition( + name="acme.mix", configuration=_HoldsV2DataTypes, size="static" ), "ZarrV2DataTypeDeclaration", ), ( - lambda: CodecDefinition( + lambda: ArrayArrayCodecDefinition( name="acme.mix", configuration=_HoldsANestedV2DataType, - kind="array_array", size="static", ), "ZarrV2DataTypeDeclaration", @@ -443,7 +440,8 @@ def test_error_a_declaration_read_by_hand_by_a_definition_filed_under_another_na def test_error_a_declaration_refused_by_hand_by_a_definition_of_another_kind() -> None: with pytest.raises( - TypeError, match="read as a DataTypeDefinition is read by one, got CodecDefinition" + TypeError, + match="read as a DataTypeDefinition is read by one, got BytesBytesCodecDefinition", ): RefusedDeclaration( json="gzip", name="gzip", read_as=DataTypeDefinition, definition=GZIP_CODEC @@ -1148,10 +1146,9 @@ def test_read_configuration_is_the_check_and_then_the_rules() -> None: def test_a_scope_takes_a_name_over() -> None: - strict = CodecDefinition( + strict = BytesBytesCodecDefinition( name="gzip", configuration=GzipCodecConfiguration, - kind="bytes_bytes", size="dynamic", rules=lambda configuration, nested: ( [ValidationProblem(("level",), "level 0 stores uncompressed", "invalid_value")] @@ -1335,7 +1332,7 @@ def test_a_declaration_copied_or_pickled_is_read_by_a_definition_equal_to_its_ow def test_a_definition_is_shown_by_its_kind_and_name() -> None: # Kept short, because a reading that holds the definition shows it this way. - assert repr(GZIP_CODEC) == "CodecDefinition(name='gzip')" + assert repr(GZIP_CODEC) == "BytesBytesCodecDefinition(name='gzip')" assert repr(INT8_DATA_TYPE) == "DataTypeDefinition(name='int8')" @@ -1366,29 +1363,26 @@ class AcmeCodecDefinition(CodecDefinition[Any]): def test_error_a_definition_configuration_is_a_typeddict() -> None: with pytest.raises(TypeError, match="give the TypedDict"): - CodecDefinition(name="acme.bad", configuration=dict, kind="bytes_bytes", size="dynamic") + BytesBytesCodecDefinition(name="acme.bad", configuration=dict, size="dynamic") def test_error_a_definition_member_is_a_shape_json_takes() -> None: with pytest.raises(TypeError, match="Unreadable: members is not a shape JSON takes"): - CodecDefinition( - name="acme.bad", configuration=Unreadable, kind="bytes_bytes", size="dynamic" - ) + BytesBytesCodecDefinition(name="acme.bad", configuration=Unreadable, size="dynamic") def test_error_a_configuration_says_what_its_other_keys_are() -> None: # An open TypedDict would accept a misspelled key without reporting it. with pytest.raises(TypeError, match="Loose says nothing of the keys it does not declare"): - CodecDefinition(name="acme.loose", configuration=Loose, kind="bytes_bytes", size="dynamic") + BytesBytesCodecDefinition(name="acme.loose", configuration=Loose, size="dynamic") def test_error_a_member_typed_as_plain_declaration_json_is_refused() -> None: # It would be checked only as JSON, and its name would never be matched to a definition. with pytest.raises(TypeError, match="annotate it with the declaration alias of its kind"): - CodecDefinition( + BytesBytesCodecDefinition( name="acme.plain", configuration=AcmePlainConfiguration, - kind="bytes_bytes", size="dynamic", ) @@ -1396,10 +1390,9 @@ def test_error_a_member_typed_as_plain_declaration_json_is_refused() -> None: def test_error_a_configuration_whose_annotations_do_not_resolve() -> None: # The error names the TypedDict that holds the annotation. with pytest.raises(TypeError, match="AcmeDecimal: name 'Decimal' is not defined"): - CodecDefinition( + BytesBytesCodecDefinition( name="acme.unresolved", configuration=AcmeUnresolvedConfiguration, - kind="bytes_bytes", size="dynamic", ) @@ -1417,10 +1410,9 @@ def test_error_a_configuration_with_a_constraint_the_checker_does_not_read() -> r".*; check it in the definition's `rules`" ), ): - CodecDefinition( + BytesBytesCodecDefinition( name="acme.unchecked", configuration=AcmeUncheckedConfiguration, - kind="bytes_bytes", size="dynamic", ) @@ -1434,21 +1426,17 @@ def test_error_a_fill_value_with_a_constraint_its_type_cannot_take() -> None: def test_error_a_definition_name_is_a_string() -> None: with pytest.raises(TypeError, match="a definition's name is a string"): - CodecDefinition( + BytesBytesCodecDefinition( name=5, # pyright: ignore[reportArgumentType] configuration=EmptyConfiguration, - kind="bytes_bytes", size="dynamic", ) -def test_error_a_codec_kind_is_one_of_three() -> None: - with pytest.raises(TypeError, match="kind is one of"): - CodecDefinition( - name="acme.k", - configuration=EmptyConfiguration, - kind="bytes_to_array", # pyright: ignore[reportArgumentType] - size="dynamic", +def test_error_a_codec_is_built_as_one_of_its_kinds() -> None: + with pytest.raises(TypeError, match="abstract class CodecDefinition"): + CodecDefinition( # pyright: ignore[reportAbstractUsage] + name="acme.k", configuration=EmptyConfiguration, size="dynamic" ) @@ -1484,16 +1472,16 @@ def test_error_a_data_type_fill_value_holding_a_declaration() -> None: (DataTypeDefinition, "storage"), (ChunkGridDefinition, "shape_rules"), (ChunkGridDefinition, "chunk_lengths"), - (CodecDefinition, "chunk_rules"), - (CodecDefinition, "transition"), - (CodecDefinition, "pipelines"), + (ArrayArrayCodecDefinition, "chunk_rules"), + (ArrayArrayCodecDefinition, "transition"), + (ArrayArrayCodecDefinition, "pipelines"), ], ) def test_error_a_function_member_that_is_not_a_function( kind: type[Definition[Any]], member: str ) -> None: # Each member that the definition's annotations declare as a `Callable`. - codec = {"kind": "array_array", "size": "static"} if kind is CodecDefinition else {} + codec = {"size": "static"} if issubclass(kind, CodecDefinition) else {} with pytest.raises(TypeError, match=f"'acme.t': {member} is a function, got 'none'"): kind(name="acme.t", configuration=EmptyConfiguration, **codec, **{member: "none"}) # pyright: ignore[reportArgumentType] @@ -1575,9 +1563,7 @@ def test_error_a_bad_name_is_a_problem_when_the_declaration_is_not_json_too() -> def test_error_a_definition_is_named_as_the_spec_names_an_extension(name: str) -> None: # No document could use this name, so the definition would never read anything. with pytest.raises(TypeError, match="so no document names it"): - CodecDefinition( - name=name, configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" - ) + BytesBytesCodecDefinition(name=name, configuration=EmptyConfiguration, size="dynamic") def test_error_a_declaration_built_by_hand_is_named_as_the_spec_names_one() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_kinds.py b/packages/zarr-metadata/tests/v3/test_kinds.py index 1939347603..2638e49e1c 100644 --- a/packages/zarr-metadata/tests/v3/test_kinds.py +++ b/packages/zarr-metadata/tests/v3/test_kinds.py @@ -20,6 +20,7 @@ from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context, ZarrV2DataTypeDefinition from zarr_metadata.v3.definition import ( CORE_CONTEXT, + BytesBytesCodecDefinition, CodecDefinition, DataTypeDefinition, ZarrV3Context, @@ -27,7 +28,7 @@ @dataclass(frozen=True, kw_only=True, slots=True, repr=False) -class MyCodec(CodecDefinition[Any]): +class MyCodec(BytesBytesCodecDefinition[Any]): """A codec definition with an extra attribute of its own. It is still a codec.""" note: str = "" @@ -40,7 +41,7 @@ class NoKind(Definition[Any]): def test_a_subclass_of_a_kind_is_a_definition_of_that_kind() -> None: """A definition built as a subclass of `CodecDefinition` is filed, read and named as a codec: the kind is the nearest class in the MRO declared with `kind=True`, not the class of the definition itself.""" - mine = MyCodec(name="mine", configuration=EmptyConfiguration, kind="bytes_bytes", size="static") + mine = MyCodec(name="mine", configuration=EmptyConfiguration, size="static") assert kind_of(mine) is CodecDefinition context = ZarrV3Context(codecs=[mine]) assert context.claimant(CodecDefinition, "mine") is mine diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 8e73731d38..cfde42f277 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -28,6 +28,9 @@ ) from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, + ArrayArrayCodecDefinition, + ArrayBytesCodecDefinition, + BytesBytesCodecDefinition, Chunk, CodecDefinition, DataTypeDefinition, @@ -51,17 +54,16 @@ def _tiled(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> C return Chunk(doubled, chunk.data_type) -ACME_TILE = CodecDefinition( +ACME_TILE = ArrayArrayCodecDefinition( name="acme.tile", configuration=EmptyConfiguration, - kind="array_array", size="static", transition=_tiled, ) """An array -> array codec that declares its output chunk.""" -ACME_SHUFFLE = CodecDefinition( - name="acme.shuffle", configuration=EmptyConfiguration, kind="array_array", size="static" +ACME_SHUFFLE = ArrayArrayCodecDefinition( + name="acme.shuffle", configuration=EmptyConfiguration, size="static" ) """An array -> array codec that does not declare its output chunk.""" @@ -360,10 +362,9 @@ def test_error_a_scale_or_offset_that_is_not_a_value_of_the_data_type_it_is_hand def test_error_a_transition_that_gives_something_else() -> None: - lying = CodecDefinition( + lying = ArrayArrayCodecDefinition( name="acme.lying", configuration=EmptyConfiguration, - kind="array_array", size="static", transition=lambda configuration, nested, chunk: "a chunk", # pyright: ignore[reportArgumentType] ) @@ -376,10 +377,9 @@ def test_error_a_transition_that_raises_says_whose_it_is() -> None: def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> Chunk: raise KeyError("axis") - raising = CodecDefinition( + raising = ArrayArrayCodecDefinition( name="acme.raising", configuration=EmptyConfiguration, - kind="array_array", size="static", transition=transition, ) @@ -409,10 +409,9 @@ def test_error_a_transition_that_builds_a_chunk_of_something_else( def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> Chunk: return Chunk(lengths, data_type) # pyright: ignore[reportArgumentType] - odd = CodecDefinition( + odd = ArrayArrayCodecDefinition( name="acme.odd", configuration=EmptyConfiguration, - kind="array_array", size="static", transition=transition, ) @@ -427,24 +426,20 @@ def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) @pytest.mark.parametrize( ("kind", "attribute"), [ - ("bytes_bytes", "chunk_rules"), - ("bytes_bytes", "transition"), - ("bytes_bytes", "pipelines"), - ("array_bytes", "transition"), + (BytesBytesCodecDefinition, "chunk_rules"), + (BytesBytesCodecDefinition, "transition"), + (BytesBytesCodecDefinition, "pipelines"), + (ArrayBytesCodecDefinition, "transition"), ], ) -def test_error_a_codec_hook_nothing_would_ask(kind: str, attribute: str) -> None: +def test_error_a_codec_hook_nothing_would_ask( + kind: type[CodecDefinition[Any]], attribute: str +) -> None: # A bytes -> bytes codec receives bytes, and only an array -> array - # codec outputs a chunk, so these hooks would never be called. + # codec outputs a chunk, so their kinds have no such hooks. hook = {attribute: lambda configuration, nested, chunk: ()} - with pytest.raises(TypeError, match=f"'acme.x': {attribute}, "): - CodecDefinition( - name="acme.x", - configuration=EmptyConfiguration, - kind=kind, # pyright: ignore[reportArgumentType] - size="static", - **hook, # pyright: ignore[reportArgumentType] - ) + with pytest.raises(TypeError, match=f"unexpected keyword argument '{attribute}'"): + kind(name="acme.x", configuration=EmptyConfiguration, size="static", **hook) # pyright: ignore[reportArgumentType] def test_an_array_document_s_codecs_are_each_judged_against_the_chunk_they_are_handed() -> None: @@ -489,10 +484,9 @@ def _holder( pipelines: object, types: JSONValue = ("uint8",) ) -> ResolvedDeclaration[CodecDefinition[Any]]: """A codec that holds a pipeline of codecs and a list of data types, `types`, and whose `pipelines` hook is `pipelines`.""" - holder = CodecDefinition( + holder = ArrayBytesCodecDefinition( name="acme.holder", configuration=AcmeHolderConfiguration, - kind="array_bytes", size="dynamic", pipelines=pipelines, # pyright: ignore[reportArgumentType] ) diff --git a/packages/zarr-metadata/tests/v3/test_scope.py b/packages/zarr-metadata/tests/v3/test_scope.py index 1feca41d63..5ccc6b425d 100644 --- a/packages/zarr-metadata/tests/v3/test_scope.py +++ b/packages/zarr-metadata/tests/v3/test_scope.py @@ -32,6 +32,7 @@ CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, ArrayExtraFieldDefinition, + BytesBytesCodecDefinition, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, @@ -50,9 +51,7 @@ }, } -MY_GZIP = CodecDefinition( - name="gzip", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" -) +MY_GZIP = BytesBytesCodecDefinition(name="gzip", configuration=EmptyConfiguration, size="dynamic") """A private definition of the name `gzip`: a second definition under the same name, which takes no configuration. Definitions compare by their contents, so a definition rebuilt from the @@ -109,9 +108,9 @@ def test_error_a_scope_conflict_says_each_disagreement() -> None: ) assert error.conflicts[0].key == (CodecDefinition, "bytes") assert str(error) == ( - "codec 'bytes' at ('codecs', 0): claimed CodecDefinition(name='bytes') of " + "codec 'bytes' at ('codecs', 0): claimed ArrayBytesCodecDefinition(name='bytes') of " "BytesCodecConfiguration, found no definition; " - "codec 'gzip': claimed no definition, found CodecDefinition(name='gzip') of " + "codec 'gzip': claimed no definition, found BytesBytesCodecDefinition(name='gzip') of " "GzipCodecConfiguration" ) @@ -448,9 +447,7 @@ def test_a_gain_is_judged_by_what_the_definition_reads_not_by_spelling() -> None assert refines(read, unread) -OTHER_GZIP = CodecDefinition( - name="gzip", configuration=EmptyConfiguration, kind="bytes_bytes", size="static" -) +OTHER_GZIP = BytesBytesCodecDefinition(name="gzip", configuration=EmptyConfiguration, size="static") """A second definition under the core gzip's name, which makes a union conflict.""" From 8d8e730ecba315b36f87bd72e0cde006dc2faa14 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 12 Nov 2026 17:44:17 +0100 Subject: [PATCH 05/11] feat(zarr-metadata): let each data type declare its number class `cast_value` and `scale_offset` decided which data types they accept from lists of data type names: 25 names across three sets, 11 of them float types this package does not define, kept only so `wrap` could refuse them. A data type defined elsewhere was judged by whether its name happened to be listed. A `DataTypeDefinition` now declares what kind of numbers its values are with `number_class`, a `NumberClass`: integral, floating-point, complex or non-numeric. The two codecs' rules read it, as the `bytes` codec reads `storage`: `cast_value` refuses complex and non-numeric data types on either side of the cast and wraps only to one that is not floating-point, and `scale_offset` refuses non-numeric ones. This is how their specs state it ("data types that model real numbers", "an integral type"), and neither a codec nor a data type has to name the other, so an extension codec can read the class of a core data type and an extension data type is judged by what it declares. A data type that declares no number class is accepted, as an unknown `storage` is, so nothing that was accepted is refused. Each of the 24 data types this package defines declares its class and is accepted or refused as before, with the same messages. Assisted-by: ClaudeCode:claude-fable-5-2 --- .../zarr-metadata/changes/XXXX.feature.md | 15 ++ packages/zarr-metadata/docs/extensions.md | 7 +- .../src/zarr_metadata/testing/_codec.py | 25 ++- .../src/zarr_metadata/v3/_definition.py | 31 ++++ .../src/zarr_metadata/v3/codec/_arithmetic.py | 83 ++------- .../src/zarr_metadata/v3/codec/cast_value.py | 19 +- .../zarr_metadata/v3/codec/scale_offset.py | 2 +- .../src/zarr_metadata/v3/data_type/bool.py | 1 + .../src/zarr_metadata/v3/data_type/bytes.py | 1 + .../zarr_metadata/v3/data_type/complex128.py | 1 + .../zarr_metadata/v3/data_type/complex64.py | 1 + .../v3/data_type/fixed_length_utf32.py | 1 + .../src/zarr_metadata/v3/data_type/float16.py | 1 + .../src/zarr_metadata/v3/data_type/float32.py | 1 + .../src/zarr_metadata/v3/data_type/float64.py | 1 + .../src/zarr_metadata/v3/data_type/int16.py | 1 + .../src/zarr_metadata/v3/data_type/int32.py | 1 + .../src/zarr_metadata/v3/data_type/int64.py | 1 + .../src/zarr_metadata/v3/data_type/int8.py | 1 + .../v3/data_type/null_terminated_bytes.py | 1 + .../v3/data_type/numpy_datetime64.py | 8 +- .../v3/data_type/numpy_timedelta64.py | 8 +- .../src/zarr_metadata/v3/data_type/raw.py | 1 + .../src/zarr_metadata/v3/data_type/string.py | 1 + .../src/zarr_metadata/v3/data_type/struct.py | 1 + .../src/zarr_metadata/v3/data_type/uint16.py | 1 + .../src/zarr_metadata/v3/data_type/uint32.py | 1 + .../src/zarr_metadata/v3/data_type/uint64.py | 1 + .../src/zarr_metadata/v3/data_type/uint8.py | 1 + .../v3/data_type/variable_length_bytes.py | 1 + .../v3/data_type/zarr_python_raw_bytes.py | 1 + .../src/zarr_metadata/v3/definition.py | 8 + .../zarr-metadata/tests/test_public_api.py | 2 + .../tests/v3/test_number_class.py | 162 ++++++++++++++++++ 34 files changed, 305 insertions(+), 87 deletions(-) create mode 100644 packages/zarr-metadata/tests/v3/test_number_class.py diff --git a/packages/zarr-metadata/changes/XXXX.feature.md b/packages/zarr-metadata/changes/XXXX.feature.md index cb9e5918d4..b2b34e7708 100644 --- a/packages/zarr-metadata/changes/XXXX.feature.md +++ b/packages/zarr-metadata/changes/XXXX.feature.md @@ -7,3 +7,18 @@ receives an array, an `ArrayCodecDefinition`, has `chunk_rules` and `pipelines`, an array -> array codec also has `transition`, and a bytes -> bytes codec, which receives bytes, has none of them, so building one with any of them is a `TypeError` from the class itself. + +A v3 data type definition declares what kind of numbers its values are with +`number_class`, a `NumberClass`: `"integral"`, `"floating_point"`, +`"complex"` or `"non_numeric"`. `cast_value` and `scale_offset` read it to +decide which data types they accept, where each kept a list of data type +names, so a data type defined elsewhere is judged by what it declares: +`cast_value` refuses one that declares complex or non-numeric values, on +either side of the cast, and refuses to `wrap` to one that declares +floating-point values, and `scale_offset` refuses one that declares +non-numeric values. A data type that declares no number class is accepted by +both, as before. Every data type this package defines declares its own, and is +accepted or refused as before. A data type defined elsewhere under one of the +float names the `cast_value` spec lists, such as `bfloat16`, was refused as a +`wrap` target by its name, and is now refused once it declares +`"floating_point"`. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index 91305dfd96..96011f1626 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -109,7 +109,12 @@ Other kinds declare more: `create_default` is given no codecs (`array_bytes_codec_default`, such as `vlen-utf8` for `string`). A data type whose values vary in size declares its own, since the default, `bytes`, encodes values of a fixed - size. + size. It declares what kind of numbers its values are (`number_class`: + `"integral"`, `"floating_point"`, `"complex"` or `"non_numeric"`), which + the codecs that are defined only for some kinds of numbers read: + `cast_value` refuses a data type that does not model real numbers, and + `scale_offset` one whose values are not numbers. A data type that does + not declare it is accepted by both. - A chunk grid declares which shapes it fits (`shape_rules`) and its chunk lengths (`chunk_lengths`). - A codec is defined by the class of its kind, `ArrayArrayCodecDefinition`, diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index 36cbd5b1d4..11e8ac8c89 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -35,10 +35,11 @@ fill_values_of, pipelines_for, read, + scope_of, ) from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, MAX_RANK, CodecStrategy from zarr_metadata.v3._definition import Chunk, DataTypeDefinition, storage_of -from zarr_metadata.v3.codec._arithmetic import FLOATING_POINT, NO_REAL_NUMBERS, NOT_NUMBERS +from zarr_metadata.v3.codec._arithmetic import NO_REAL_NUMBERS, NOT_NUMBERS from zarr_metadata.v3.codec.blosc import ( BLOSC_CNAME, BLOSC_CODEC, @@ -86,6 +87,7 @@ CodecDefinition, CodecSize, Lengths, + NumberClass, ) _BLOSC_MAX_TYPESIZE: Final = 255 @@ -210,7 +212,7 @@ def _scale_offset( if definition is None: offset: st.SearchStrategy[JSONValue] = _SMALL_NUMBERS scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) - elif definition.name in NOT_NUMBERS: + elif definition.number_class in NOT_NUMBERS: return None elif definition in INTEGER_RANGES or definition in _NUMPY_TIMES: low, high = INTEGER_RANGES.get(definition, _NUMPY_TIME_RANGE) @@ -256,9 +258,11 @@ def _unchunked(configurations: Configurations) -> CodecStrategy: return CodecStrategy(configurations=lambda _chunk, _strategies: configurations) -def _name_read(data_type: ResolvedDeclaration[DataTypeDefinition[Any]] | None) -> str | None: - """The name of the definition that accepted `data_type`; None when none did.""" - return data_type.definition.name if isinstance(data_type, AcceptedDeclaration) else None +def _number_class_read( + data_type: ResolvedDeclaration[DataTypeDefinition[Any]] | None, +) -> NumberClass | None: + """The number class the definition that accepted `data_type` declares; None when none accepted it, or it declares none.""" + return data_type.definition.number_class if isinstance(data_type, AcceptedDeclaration) else None def _source_scalars(chunk: Chunk, strategies: Strategies) -> st.SearchStrategy[JSONValue]: @@ -299,7 +303,7 @@ def _cast_value_configuration( declaration, DataTypeDefinition, strategies ) modes: tuple[CastOutOfRangeMode, ...] = CAST_OUT_OF_RANGE_MODE - if _name_read(target) in FLOATING_POINT: + if _number_class_read(target) == "floating_point": modes = tuple(mode for mode in modes if mode != "wrap") configuration = CastValueCodecConfiguration(data_type=declaration) if draw(st.booleans()): @@ -325,9 +329,14 @@ def _cast_value( of its direction; `wrap` is drawn only for a type that is not floating-point. """ - if _name_read(chunk.data_type) in NO_REAL_NUMBERS: + if _number_class_read(chunk.data_type) in NO_REAL_NUMBERS: return None - targets = [name for name in data_type_names(strategies) if name not in NO_REAL_NUMBERS] + data_types = scope_of(strategies).data_types + targets = [ + name + for name in data_type_names(strategies) + if data_types[name].number_class not in NO_REAL_NUMBERS + ] if len(targets) == 0: return None return _cast_value_configuration(targets, _source_scalars(chunk, strategies), strategies) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index 6100aa7df7..39f2c08b8d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -127,6 +127,21 @@ def unknown_storage(*_: object) -> None: """The storage of a data type that does not declare it: unknown.""" +NumberClass = Literal["integral", "floating_point", "complex", "non_numeric"] +"""What kind of numbers a data type's values are: integers, floating-point numbers, complex numbers, or not numbers at all. + +A codec that is defined only for some kinds of numbers reads this. For +example, `cast_value` "is only defined for data types that model real +numbers: floating-point and integral data types" +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L5-L9), +and its `wrap` mode is "Only permitted when data_type is an integral +type" +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L84). +Values that are truth values, text, byte strings or records are +`non_numeric`. +""" + + def little_endian_bytes_codec(*_: object) -> BytesCodecObject: """The array -> bytes codec of a data type that does not declare one: `bytes` with little `endian`, which encodes values of any fixed size.""" return {"name": BYTES_CODEC_NAME, "configuration": {"endian": "little"}} @@ -197,6 +212,13 @@ class DataTypeDefinition(WithFillValue[C], kind=True, format=3): any fixed size, so a data type whose values vary in size declares its own. + `number_class` declares what kind of numbers the values are: integral, + floating-point, complex, or not numbers. A codec that is defined only + for some kinds of numbers reads it, as `cast_value` reads it to accept + only data types that model real numbers. A data type that does not + declare it leaves its number class unknown, and such a codec accepts + it, since nothing is known of its values. + A data type named the way a document writes raw bits of one size (such as `r16`) is rejected. That name is read as `r*`, so this definition would never be used. @@ -235,8 +257,16 @@ def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JS little_endian_bytes_codec ) """The array -> bytes codec that encodes the values of an array built without codecs, such as `vlen-utf8` for `string`.""" + number_class: NumberClass | None = None + """What kind of numbers the values are, such as `"integral"` for `int8`; None when unknown.""" def _refusal(self) -> str | None: + number_class = cast("object", self.number_class) + if number_class is not None and number_class not in get_args(NumberClass): + return ( + f"{self.name!r}: number_class is one of {get_args(NumberClass)!r} or None, " + f"got {number_class!r}" + ) if RAW_BYTES_NAME_PATTERN.fullmatch(self.name) is not None: return ( f"{self.name!r} is how a document writes raw bits of one size, which read as " @@ -836,6 +866,7 @@ def chunk_grid_lengths( "ExtraFieldDefinition", "GroupExtraFieldDefinition", "Lengths", + "NumberClass", "StorageClass", "StorageTransformerDefinition", "chunk_grid_lengths", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py index f56da6052e..0f11e99e39 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py @@ -1,85 +1,32 @@ -"""The data types that the arithmetic codecs, `cast_value` and `scale_offset`, accept. +"""The number classes that the arithmetic codecs, `cast_value` and `scale_offset`, refuse. `cast_value` "is only defined for data types that model real numbers: floating-point and integral data types" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L5-L9), and `scale_offset` "for data types where multiplication, division, addition, and subtraction are well-defined" -(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9); -Each spec then lists "the following data types defined in this -repository", and the list is the same for both. A data type defined -elsewhere may also qualify, so a data type that the list does not name -is rejected only when the spec text itself excludes it. - -Neither codec rejects the numpy time types. Each of them "is compatible -with any codec that supports arrays of signed 64-bit integers" -(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120), -and both codecs support those, even though neither list names the time -types. +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9). +Each data type says what kind of numbers its values are with its +`number_class`, and each codec refuses the classes its spec excludes. A +data type that does not declare its number class is accepted, since +nothing is known of its values. """ from typing import Final -from zarr_metadata.v3._definition import RAW_BYTES_NAME -from zarr_metadata.v3._types.data_type import ( - BOOL_DATA_TYPE_NAME, - BYTES_DATA_TYPE_NAME, - COMPLEX64_DATA_TYPE_NAME, - COMPLEX128_DATA_TYPE_NAME, - FIXED_LENGTH_UTF32_DATA_TYPE_NAME, - NULL_TERMINATED_BYTES_DATA_TYPE_NAME, - STRING_DATA_TYPE_NAME, - STRUCT_DATA_TYPE_NAME, - VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME, - ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME, -) - -FLOATING_POINT: Final = frozenset( - { - "float4_e2m1fn", - "float6_e2m3fn", - "float6_e3m2fn", - "float8_e3m4", - "float8_e4m3", - "float8_e4m3b11fnuz", - "float8_e4m3fnuz", - "float8_e5m2", - "float8_e5m2fnuz", - "float8_e8m0fnu", - "bfloat16", - "float16", - "float32", - "float64", - } -) -"""The floating-point data types that the lists name. None of them is integral, so `cast_value` may not wrap to them.""" +from zarr_metadata.v3._definition import NumberClass -NOT_NUMBERS: Final = frozenset( - { - BOOL_DATA_TYPE_NAME, - RAW_BYTES_NAME, - STRING_DATA_TYPE_NAME, - BYTES_DATA_TYPE_NAME, - STRUCT_DATA_TYPE_NAME, - FIXED_LENGTH_UTF32_DATA_TYPE_NAME, - NULL_TERMINATED_BYTES_DATA_TYPE_NAME, - VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME, - ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME, - } -) -"""Data types whose values are not numbers (truth values, raw bits, text, byte strings, records). Neither codec accepts them.""" +NOT_NUMBERS: Final[frozenset[NumberClass]] = frozenset({"non_numeric"}) +"""The number class of values that are not numbers (truth values, raw bits, text, byte strings, records). Neither codec accepts it.""" -COMPLEX: Final = frozenset({COMPLEX64_DATA_TYPE_NAME, COMPLEX128_DATA_TYPE_NAME}) -"""Complex data types. They do not model real numbers, so `cast_value` does not accept them. +NO_REAL_NUMBERS: Final[frozenset[NumberClass]] = frozenset({"non_numeric", "complex"}) +"""The number classes that model no real number, which `cast_value` takes neither from nor to. Addition, subtraction, multiplication and division are well-defined on -them, but the `scale_offset` spec does not name them in its list. So the -spec leaves open whether `scale_offset` accepts them. +complex numbers, but the `scale_offset` spec does not name the complex +data types in its list. So the spec leaves open whether `scale_offset` +accepts them, and it does here. """ -NO_REAL_NUMBERS: Final = NOT_NUMBERS | COMPLEX -"""The data types that model no real number, which `cast_value` takes neither from nor to.""" - - -__all__ = ["COMPLEX", "FLOATING_POINT", "NOT_NUMBERS", "NO_REAL_NUMBERS"] +__all__ = ["NOT_NUMBERS", "NO_REAL_NUMBERS"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 2520f9e2eb..377c92dfc1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -17,6 +17,7 @@ from zarr_metadata.v3._definition import ( ArrayArrayCodecDefinition, Chunk, + NumberClass, ) from zarr_metadata.v3._types.codec import ( CAST_OUT_OF_RANGE_MODE, @@ -31,7 +32,7 @@ ScalarMap, ScalarMapEntry, ) -from zarr_metadata.v3.codec._arithmetic import FLOATING_POINT, NO_REAL_NUMBERS +from zarr_metadata.v3.codec._arithmetic import NO_REAL_NUMBERS def _scalars( @@ -68,21 +69,25 @@ def _rules( (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L84); and a scalar of the target type is encoded with its fill value encoding. These rules check the target data type, which the scope - read. The chunk rules check the input data type. If the target does - not model real numbers, that is the only problem reported for it. + read, by the number class its definition declares. The chunk rules + check the input data type. If the target does not model real numbers, + that is the only problem reported for it. A target that does not + declare its number class is accepted, since nothing is known of its + values. """ target = nested.get(("data_type",)) if not isinstance(target, AcceptedDeclaration): return - name, written = target.definition.name, target.name - if name in NO_REAL_NUMBERS: + number_class: NumberClass | None = target.definition.number_class + written = target.name + if number_class in NO_REAL_NUMBERS: yield ValidationProblem( ("data_type",), f"expected a data type that models real numbers, got {shown(written)}", "invalid_value", ) return - if configuration.get("out_of_range") == "wrap" and name in FLOATING_POINT: + if configuration.get("out_of_range") == "wrap" and number_class == "floating_point": yield ValidationProblem( ("out_of_range",), f"expected an integral data_type to wrap to, got {shown(written)}", @@ -106,7 +111,7 @@ def _chunk_rules( source = chunk.data_type if not isinstance(source, AcceptedDeclaration): return - if source.definition.name in NO_REAL_NUMBERS: + if source.definition.number_class in NO_REAL_NUMBERS: yield ValidationProblem( (), f"expected a chunk of a data type that models real numbers, got {shown(source.name)}", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 6b9782e345..b5c10f61c0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -59,7 +59,7 @@ def _chunk_rules( source = chunk.data_type if not isinstance(source, AcceptedDeclaration): return - if source.definition.name in NOT_NUMBERS: + if source.definition.number_class in NOT_NUMBERS: yield ValidationProblem( (), f"expected a chunk of a data type with arithmetic, got {shown(source.name)}", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index df008b6710..fd309779d1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -20,6 +20,7 @@ fill_value=BoolFillValue, fill_value_default=ConstantFillValue(False), storage=single_byte, + number_class="non_numeric", ) """The `bool` data type. It is written as a bare name and has no configuration.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index 3aea0c2663..ca402ee052 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -52,6 +52,7 @@ def _fill_value_canonical( fill_value_canonical=_fill_value_canonical, storage=variable_length, array_bytes_codec_default=vlen_bytes_codec, + number_class="non_numeric", ) """The `bytes` data type. It is written as a bare name and has no configuration. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py index 40cb7d09df..43a70bd7d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py @@ -32,6 +32,7 @@ fill_value_rules=complex_fill_value_rules(FLOAT64_DATA_TYPE.fill_value_rules), fill_value_canonical=complex_fill_value_canonical(FLOAT64_DATA_TYPE.fill_value_canonical), storage=multi_byte, + number_class="complex", ) """The `complex128` data type. It is written as a bare name and has no configuration. Its fill value is a pair of `float64` components.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py index 55281b4973..e8ba97aab0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py @@ -32,6 +32,7 @@ fill_value_rules=complex_fill_value_rules(FLOAT32_DATA_TYPE.fill_value_rules), fill_value_canonical=complex_fill_value_canonical(FLOAT32_DATA_TYPE.fill_value_canonical), storage=multi_byte, + number_class="complex", ) """The `complex64` data type. It is written as a bare name and has no configuration. Its fill value is a pair of `float32` components.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py index b3e23251b7..7bb8c135e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py @@ -79,6 +79,7 @@ def _fill_value_canonical( fill_value_rules=_fill_value_rules, fill_value_canonical=_fill_value_canonical, storage=multi_byte, + number_class="non_numeric", ) """The `fixed_length_utf32` data type: NumPy's `U`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 4fe61579f8..27725e8925 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -47,6 +47,7 @@ def hex_float16(value: str) -> HexFloat16: fill_value_rules=float_fill_value_rules("float16", hex_float16), fill_value_canonical=float_fill_value_canonical(16), storage=multi_byte, + number_class="floating_point", ) """The `float16` data type. It is written as a bare name and has no configuration. Its fill value is a number, a named value or a hex string.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 9666121d80..d265b3d69f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -47,6 +47,7 @@ def hex_float32(value: str) -> HexFloat32: fill_value_rules=float_fill_value_rules("float32", hex_float32), fill_value_canonical=float_fill_value_canonical(32), storage=multi_byte, + number_class="floating_point", ) """The `float32` data type. It is written as a bare name and has no configuration. Its fill value is a number, a named value or a hex string.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index 69ccab10fc..db50d6a14f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -47,6 +47,7 @@ def hex_float64(value: str) -> HexFloat64: fill_value_rules=float_fill_value_rules("float64", hex_float64), fill_value_canonical=float_fill_value_canonical(64), storage=multi_byte, + number_class="floating_point", ) """The `float64` data type. It is written as a bare name and has no configuration. Its fill value is a number, a named value or a hex string.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py index 1fde31deb6..5788f40a2c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py @@ -24,6 +24,7 @@ fill_value=Int16FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `int16` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py index 496215175b..3da6dc94b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py @@ -24,6 +24,7 @@ fill_value=Int32FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `int32` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py index f0cf13d87a..136258c74b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py @@ -24,6 +24,7 @@ fill_value=Int64FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `int64` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py index 41f1a16e9c..809b72568f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py @@ -20,6 +20,7 @@ fill_value=Int8FillValue, fill_value_default=ConstantFillValue(0), storage=single_byte, + number_class="integral", ) """The `int8` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py index c5308a44ef..333f362cb5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py @@ -81,6 +81,7 @@ def _fill_value_canonical( fill_value_rules=_fill_value_rules, fill_value_canonical=_fill_value_canonical, storage=single_byte, + number_class="non_numeric", ) """zarr-python's `null_terminated_bytes` data type: NumPy's `S`. Each value is `length_bytes` single bytes.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 8ce3418c43..9a107fd521 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -29,8 +29,14 @@ fill_value_default=ConstantFillValue("NaT"), fill_value_canonical=numpy_time_fill_value_canonical, storage=numpy_time_storage, + number_class="integral", ) -"""The `numpy.datetime64` data type.""" +"""The `numpy.datetime64` data type. + +Its number class is integral, since it "is compatible with any codec that +supports arrays of signed 64-bit integers" +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). +""" __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index b78d0cafb9..90e8c628e8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -27,8 +27,14 @@ fill_value_default=ConstantFillValue("NaT"), fill_value_canonical=numpy_time_fill_value_canonical, storage=numpy_time_storage, + number_class="integral", ) -"""The `numpy.timedelta64` data type.""" +"""The `numpy.timedelta64` data type. + +Its number class is integral: like `numpy.datetime64`, it is stored as +signed 64-bit integers, and is compatible with any codec that supports +arrays of those. +""" __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 9cf03cc08b..63fc15e878 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -93,6 +93,7 @@ def _storage(configuration: RawBytesConfiguration, nested: Nested) -> StorageCla fill_value_default=_fill_value_default, fill_value_rules=_fill_value_rules, storage=_storage, + number_class="non_numeric", ) """Raw bits, `r*`: a single data type whose name carries its size. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index f7017d571c..f137cda8c5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -32,6 +32,7 @@ def _array_bytes_codec_default(*_: object) -> VlenUtf8CodecObject: fill_value_default=ConstantFillValue(""), storage=variable_length, array_bytes_codec_default=_array_bytes_codec_default, + number_class="non_numeric", ) """The `string` data type. It is written as a bare name and has no configuration. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index e7f1a54b27..af9fbc0038 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -164,6 +164,7 @@ def _storage(configuration: StructConfiguration, nested: Nested) -> StorageClass fill_value_rules=_fill_value_rules, fill_value_canonical=_fill_value_canonical, storage=_storage, + number_class="non_numeric", ) """The `struct` data type: a record of named fields. The type of each field is a nested declaration.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py index 808dd1792a..5129fabaa0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py @@ -24,6 +24,7 @@ fill_value=Uint16FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `uint16` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py index acda4e78b8..ede4821c21 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py @@ -24,6 +24,7 @@ fill_value=Uint32FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `uint32` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py index a6d72e473d..fa404617c2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py @@ -24,6 +24,7 @@ fill_value=Uint64FillValue, fill_value_default=ConstantFillValue(0), storage=multi_byte, + number_class="integral", ) """The `uint64` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py index 1aea655edf..339a48fe4d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py @@ -24,6 +24,7 @@ fill_value=Uint8FillValue, fill_value_default=ConstantFillValue(0), storage=single_byte, + number_class="integral", ) """The `uint8` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py index 0e4b241d9e..4715758909 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py @@ -43,6 +43,7 @@ def _fill_value_canonical( fill_value_canonical=_fill_value_canonical, storage=variable_length, array_bytes_codec_default=vlen_bytes_codec, + number_class="non_numeric", ) """zarr-python's `variable_length_bytes` data type. It is written as a bare name and has no configuration, and each value is stored in as many bytes as it needs.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py index eef9ed81f9..e1f41efcc9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py @@ -88,6 +88,7 @@ def _fill_value_default(configuration: ZarrPythonRawBytesConfiguration, nested: fill_value_rules=_fill_value_rules, fill_value_canonical=_fill_value_canonical, storage=single_byte, + number_class="non_numeric", ) """zarr-python's `raw_bytes` data type: NumPy's `V` without fields. Each value is `length_bytes` single bytes.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index b3777f190c..6d08e99e95 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -54,6 +54,12 @@ `vlen-utf8` for `string`, and `bytes` with little `endian` for a data type that does not declare it. `default_array_bytes_codec(data_type)` calls it for a data type declaration the scope read. +A data type also declares what kind of numbers its values are with +`number_class`, a `NumberClass`: integral, floating-point, complex, or +not numbers. A codec that is defined only for some kinds of numbers +reads it: `cast_value` accepts only data types that model real numbers, +and `scale_offset` only those with arithmetic. A data type that does not +declare it leaves its number class unknown, and those codecs accept it. A chunk grid declares which array shapes it fits with `shape_rules`, a function that yields the spec violations in a grid with its @@ -134,6 +140,7 @@ ExtraFieldDefinition, GroupExtraFieldDefinition, Lengths, + NumberClass, StorageClass, StorageTransformerDefinition, default_array_bytes_codec, @@ -160,6 +167,7 @@ "ExtraFieldDefinition", "GroupExtraFieldDefinition", "Lengths", + "NumberClass", "Stage", "StorageClass", "StorageTransformerDefinition", diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 926b7753d4..e09dc0a0a8 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -325,6 +325,8 @@ def test_all_is_grouped_and_unique() -> None: "Loc", "Nested", "NodeName", + # What kind of numbers a data type's values are, as `StorageClass` says how they are stored. + "NumberClass", "NodePath", # The result of reading a declaration in a scope: `AcceptedDeclaration` (read by the # definition that claims its name), `UnclaimedDeclaration`, or `RefusedDeclaration`. diff --git a/packages/zarr-metadata/tests/v3/test_number_class.py b/packages/zarr-metadata/tests/v3/test_number_class.py new file mode 100644 index 0000000000..2d35b35e62 --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_number_class.py @@ -0,0 +1,162 @@ +"""Test the number class a data type declares, and the arithmetic codecs that read it. + +`cast_value` accepts only data types that model real numbers, on either +side of the cast, and wraps only to an integral one. `scale_offset` +accepts only data types with arithmetic. Each reads the number class that +the data type's definition declares, and accepts a data type that +declares none, since nothing is known of its values. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, get_args + +import pytest + +from zarr_metadata.definition import EmptyConfiguration, resolve +from zarr_metadata.v3._pipeline import read_pipeline +from zarr_metadata.v3.definition import ( + CORE_AND_EXTENSIONS_CONTEXT, + ZARR_PYTHON_CONTEXT, + Chunk, + CodecDefinition, + DataTypeDefinition, + NumberClass, + ZarrV3Context, +) + +if TYPE_CHECKING: + from zarr_metadata.typed_json import JSONValue + +NUMBER_CLASSES = { + **dict.fromkeys( + [ + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "numpy.datetime64", + "numpy.timedelta64", + ], + "integral", + ), + **dict.fromkeys(["float16", "float32", "float64"], "floating_point"), + **dict.fromkeys(["complex64", "complex128"], "complex"), + **dict.fromkeys( + [ + "bool", + "bytes", + "fixed_length_utf32", + "null_terminated_bytes", + "r*", + "raw_bytes", + "string", + "struct", + "variable_length_bytes", + ], + "non_numeric", + ), +} +"""The number class of each data type this package defines, by the name its definition is filed under.""" + + +def test_each_data_type_declares_its_number_class() -> None: + defined = {**CORE_AND_EXTENSIONS_CONTEXT.data_types, **ZARR_PYTHON_CONTEXT.data_types} + assert {name: definition.number_class for name, definition in defined.items()} == ( + NUMBER_CLASSES + ) + + +ACME_DATA_TYPES = [ + *( + DataTypeDefinition( + name=f"acme.{number_class}", configuration=EmptyConfiguration, number_class=number_class + ) + for number_class in get_args(NumberClass) + ), + DataTypeDefinition(name="acme.unknown", configuration=EmptyConfiguration), +] +"""An extension data type of each number class, and one that declares none.""" + +CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(data_types=ACME_DATA_TYPES)) + +LITTLE: JSONValue = {"name": "bytes", "configuration": {"endian": "little"}} + + +def _cast(data_type: JSONValue, **members: JSONValue) -> JSONValue: + return {"name": "cast_value", "configuration": {"data_type": data_type, **members}} + + +def _chunk(data_type: str) -> Chunk: + return Chunk((frozenset({4}),), resolve(data_type, DataTypeDefinition, CONTEXT)[0]) + + +_NO_REALS = 'expected a chunk of a data type that models real numbers, got "{}"' +_NO_REAL_TARGET = 'expected a data type that models real numbers, got "{}"' +_NO_ARITHMETIC = 'expected a chunk of a data type with arithmetic, got "{}"' +_NO_WRAP = 'expected an integral data_type to wrap to, got "{}"' + + +@pytest.mark.parametrize( + ("codec", "data_type", "found"), + [ + # cast_value takes real numbers, and a data type nothing is known of. + (_cast("int8"), "acme.integral", []), + (_cast("int8"), "acme.floating_point", []), + (_cast("int8"), "acme.unknown", []), + (_cast("int8"), "acme.complex", [((0,), _NO_REALS.format("acme.complex"))]), + (_cast("int8"), "acme.non_numeric", [((0,), _NO_REALS.format("acme.non_numeric"))]), + # It casts to the same data types it casts from. + (_cast("acme.integral"), "int8", []), + (_cast("acme.floating_point"), "int8", []), + (_cast("acme.unknown"), "int8", []), + ( + _cast("acme.complex"), + "int8", + [((0, "configuration", "data_type"), _NO_REAL_TARGET.format("acme.complex"))], + ), + ( + _cast("acme.non_numeric"), + "int8", + [((0, "configuration", "data_type"), _NO_REAL_TARGET.format("acme.non_numeric"))], + ), + # It wraps to an integral data type, and to one nothing is known of. + (_cast("acme.integral", out_of_range="wrap"), "int8", []), + (_cast("acme.unknown", out_of_range="wrap"), "int8", []), + ( + _cast("acme.floating_point", out_of_range="wrap"), + "int8", + [((0, "configuration", "out_of_range"), _NO_WRAP.format("acme.floating_point"))], + ), + # scale_offset takes every kind of number. + ("scale_offset", "acme.integral", []), + ("scale_offset", "acme.floating_point", []), + ("scale_offset", "acme.complex", []), + ("scale_offset", "acme.unknown", []), + ("scale_offset", "acme.non_numeric", [((0,), _NO_ARITHMETIC.format("acme.non_numeric"))]), + ], +) +def test_the_arithmetic_codecs_read_the_number_class( + codec: JSONValue, data_type: str, found: list[tuple[tuple[str | int, ...], str]] +) -> None: + read = [ + resolve(each, CodecDefinition, CONTEXT, (index,)) + for index, each in enumerate([codec, LITTLE]) + ] + _, in_pipeline = read_pipeline([each for each, _ in read], _chunk(data_type)) + problems = [*(problem for _, own in read for problem in own), *in_pipeline] + assert [(problem.loc, problem.message) for problem in problems] == found + + +@pytest.mark.parametrize("number_class", ["real", 1, ("integral",)]) +def test_error_a_number_class_that_is_not_one(number_class: object) -> None: + with pytest.raises(TypeError, match="'acme.d': number_class is one of .* or None, got"): + DataTypeDefinition( + name="acme.d", + configuration=EmptyConfiguration, + number_class=number_class, # pyright: ignore[reportArgumentType] + ) From 28e0bfd0aa536dea1a485714c648312c0af07d82 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 12 Nov 2026 19:04:12 +0100 Subject: [PATCH 06/11] fix(zarr-metadata)!: refuse the data types the arithmetic codecs' specs do not list The `cast_value` and `scale_offset` specs each name the integer, unsigned integer and floating-point types as "the following data types defined in this repository" that the codec supports. That repository also defines the numpy time types and the complex types, and neither list names them. The package accepted the time types for both codecs, on the strength of the time types' own spec ("compatible with any codec that supports arrays of signed 64-bit integers"), and accepted the complex types for `scale_offset`, which its spec was read as leaving open. The codec's spec says what the codec is defined for, so both codecs now refuse `numpy.datetime64`, `numpy.timedelta64` and the complex types. The time types declare the non-numeric number class, and `scale_offset` refuses the same classes as `cast_value`, so one set serves both. zarrs gives the same verdict for `cast_value`. The strategies no longer draw these combinations, and the tests that reached them are removed or moved to the table of refused data types. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/XXXX.bugfix.md | 10 ++++ .../zarr-metadata/changes/XXXX.feature.md | 17 +++--- packages/zarr-metadata/docs/extensions.md | 6 +- .../src/zarr_metadata/testing/_codec.py | 36 +++--------- .../src/zarr_metadata/v3/_definition.py | 2 +- .../src/zarr_metadata/v3/codec/_arithmetic.py | 22 ++++---- .../zarr_metadata/v3/codec/scale_offset.py | 15 +++-- .../v3/data_type/numpy_datetime64.py | 14 +++-- .../v3/data_type/numpy_timedelta64.py | 8 +-- .../src/zarr_metadata/v3/definition.py | 7 ++- .../tests/testing/test_strategies.py | 56 ++++++------------- .../tests/v3/test_number_class.py | 14 ++--- .../zarr-metadata/tests/v3/test_pipelines.py | 35 +++++------- 13 files changed, 108 insertions(+), 134 deletions(-) create mode 100644 packages/zarr-metadata/changes/XXXX.bugfix.md diff --git a/packages/zarr-metadata/changes/XXXX.bugfix.md b/packages/zarr-metadata/changes/XXXX.bugfix.md new file mode 100644 index 0000000000..07af0924ff --- /dev/null +++ b/packages/zarr-metadata/changes/XXXX.bugfix.md @@ -0,0 +1,10 @@ +**Breaking:** `cast_value` and `scale_offset` refuse the data types their +specs do not list. Each spec names the integer and floating-point types as +"the following data types defined in this repository" that it supports, and +that repository also defines the numpy time types and the complex types. So +both codecs now refuse `numpy.datetime64` and `numpy.timedelta64`, and +`cast_value` refuses them as the data type it casts to, and `scale_offset` +now refuses `complex64` and `complex128`. All of these were accepted. The time +types' own spec says they are "compatible with any codec that supports arrays +of signed 64-bit integers", which the two codecs' specs do not bear out. +`zarr_metadata.testing.strategies` no longer draws these combinations. diff --git a/packages/zarr-metadata/changes/XXXX.feature.md b/packages/zarr-metadata/changes/XXXX.feature.md index b2b34e7708..0765e5b880 100644 --- a/packages/zarr-metadata/changes/XXXX.feature.md +++ b/packages/zarr-metadata/changes/XXXX.feature.md @@ -12,13 +12,10 @@ A v3 data type definition declares what kind of numbers its values are with `number_class`, a `NumberClass`: `"integral"`, `"floating_point"`, `"complex"` or `"non_numeric"`. `cast_value` and `scale_offset` read it to decide which data types they accept, where each kept a list of data type -names, so a data type defined elsewhere is judged by what it declares: -`cast_value` refuses one that declares complex or non-numeric values, on -either side of the cast, and refuses to `wrap` to one that declares -floating-point values, and `scale_offset` refuses one that declares -non-numeric values. A data type that declares no number class is accepted by -both, as before. Every data type this package defines declares its own, and is -accepted or refused as before. A data type defined elsewhere under one of the -float names the `cast_value` spec lists, such as `bfloat16`, was refused as a -`wrap` target by its name, and is now refused once it declares -`"floating_point"`. +names, so a data type defined elsewhere is judged by what it declares: both +codecs refuse one that declares complex or non-numeric values, and +`cast_value` refuses to `wrap` to one that declares floating-point values. A +data type that declares no number class is accepted by both, as before. A data +type defined elsewhere under one of the float names the `cast_value` spec +lists, such as `bfloat16`, was refused as a `wrap` target by its name, and is +now refused once it declares `"floating_point"`. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index 96011f1626..57acdeffbd 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -112,9 +112,9 @@ Other kinds declare more: size. It declares what kind of numbers its values are (`number_class`: `"integral"`, `"floating_point"`, `"complex"` or `"non_numeric"`), which the codecs that are defined only for some kinds of numbers read: - `cast_value` refuses a data type that does not model real numbers, and - `scale_offset` one whose values are not numbers. A data type that does - not declare it is accepted by both. + `cast_value` and `scale_offset` refuse a data type that declares + complex or non-numeric values. A data type that does not declare it is + accepted by both. - A chunk grid declares which shapes it fits (`shape_rules`) and its chunk lengths (`chunk_lengths`). - A codec is defined by the class of its kind, `ArrayArrayCodecDefinition`, diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index 11e8ac8c89..5ec45096bd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -22,7 +22,6 @@ from zarr_metadata._common import EmptyConfiguration from zarr_metadata.definition._definition import AcceptedDeclaration from zarr_metadata.testing._data_type import ( - COMPLEX_WIDTHS, FLOAT_WIDTHS, INTEGER_RANGES, float_fill_values, @@ -39,7 +38,7 @@ ) from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, MAX_RANK, CodecStrategy from zarr_metadata.v3._definition import Chunk, DataTypeDefinition, storage_of -from zarr_metadata.v3.codec._arithmetic import NO_REAL_NUMBERS, NOT_NUMBERS +from zarr_metadata.v3.codec._arithmetic import NO_REAL_NUMBERS from zarr_metadata.v3.codec.blosc import ( BLOSC_CNAME, BLOSC_CODEC, @@ -73,8 +72,6 @@ ZSTD_MIN_LEVEL, ZstdCodecConfiguration, ) -from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE -from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE if TYPE_CHECKING: from collections.abc import Mapping @@ -116,12 +113,6 @@ _SMALL_NUMBERS: Final = st.integers(0, 127) """Values every integral and floating-point data type holds: for a chunk whose data type is not known.""" -_NUMPY_TIMES: Final = frozenset({NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE}) -"""The numpy time types, each stored as a signed 64-bit integer.""" - -_NUMPY_TIME_RANGE: Final = (-(2**63) + 1, 2**63 - 1) -"""The signed 64-bit integers but the least, which numpy reads as `NaT`: the finite numpy times.""" - @st.composite def _endians(draw: st.DrawFn, *, required: bool) -> BytesCodecConfiguration: @@ -190,18 +181,14 @@ def _offsets_and_scales( def _scale_offset( chunk: Chunk, _strategies: Strategies ) -> st.SearchStrategy[ScaleOffsetCodecConfiguration] | None: - """An `offset` and a `scale`, each a value of the data type the codec is handed, the scale not zero; None for a data type whose values are not numbers. + """An `offset` and a `scale`, each a value of the data type the codec is handed, the scale not zero; None for a data type whose values are not integers or floating-point numbers. Each is a fill value of the chunk's data type (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), so a float's is in every spelling a fill value takes, a `NaN` and an - infinity among them, and a complex number's is a pair of them. The - scale is drawn non-zero in any spelling, so that decoding never divides - by zero; a scale small enough that decoding overflows is still drawn. A - numpy time is drawn as the signed 64-bit integer it is stored as, never - `NaT`, since it "is compatible with any codec that supports arrays of - signed 64-bit integers" - (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). + infinity among them. The scale is drawn non-zero in any spelling, so + that decoding never divides by zero; a scale small enough that decoding + overflows is still drawn. A data type this package does not define, whose values it cannot draw, takes only the configuration with neither, the identity. Handed a chunk whose data type is unknown, the values are small @@ -212,22 +199,15 @@ def _scale_offset( if definition is None: offset: st.SearchStrategy[JSONValue] = _SMALL_NUMBERS scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) - elif definition.number_class in NOT_NUMBERS: + elif definition.number_class in NO_REAL_NUMBERS: return None - elif definition in INTEGER_RANGES or definition in _NUMPY_TIMES: - low, high = INTEGER_RANGES.get(definition, _NUMPY_TIME_RANGE) + elif definition in INTEGER_RANGES: + low, high = INTEGER_RANGES[definition] offset = st.integers(low, high) scale = st.integers(low, high).filter(lambda value: value != 0) elif definition in FLOAT_WIDTHS: offset = float_fill_values(FLOAT_WIDTHS[definition]) scale = offset.filter(lambda value: not is_zero_float(value)) - elif definition in COMPLEX_WIDTHS: - part = float_fill_values(COMPLEX_WIDTHS[definition]) - offset = st.tuples(part, part) - # Non-zero in either part is a non-zero complex number. - scale = st.tuples(part, part).filter( - lambda pair: not (is_zero_float(pair[0]) and is_zero_float(pair[1])) - ) else: return st.builds(ScaleOffsetCodecConfiguration) return _offsets_and_scales(offset, scale) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index 39f2c08b8d..fd0b4a089b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -137,7 +137,7 @@ def unknown_storage(*_: object) -> None: and its `wrap` mode is "Only permitted when data_type is an integral type" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L84). -Values that are truth values, text, byte strings or records are +Values that are truth values, times, text, byte strings or records are `non_numeric`. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py index 0f11e99e39..d8eaa5aadc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py @@ -6,9 +6,15 @@ and `scale_offset` "for data types where multiplication, division, addition, and subtraction are well-defined" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9). +Each spec then lists "the following data types defined in this +repository" that it supports, and the list is the same for both: the +integer, unsigned integer and floating-point types. That repository also +defines the complex types and the numpy time types, and neither list +names them. So both codecs accept the same data types. + Each data type says what kind of numbers its values are with its -`number_class`, and each codec refuses the classes its spec excludes. A -data type that does not declare its number class is accepted, since +`number_class`, and both codecs refuse the classes their lists leave out. +A data type that does not declare its number class is accepted, since nothing is known of its values. """ @@ -16,17 +22,13 @@ from zarr_metadata.v3._definition import NumberClass -NOT_NUMBERS: Final[frozenset[NumberClass]] = frozenset({"non_numeric"}) -"""The number class of values that are not numbers (truth values, raw bits, text, byte strings, records). Neither codec accepts it.""" - NO_REAL_NUMBERS: Final[frozenset[NumberClass]] = frozenset({"non_numeric", "complex"}) -"""The number classes that model no real number, which `cast_value` takes neither from nor to. +"""The number classes that neither codec takes: values that are not numbers, and complex numbers. Addition, subtraction, multiplication and division are well-defined on -complex numbers, but the `scale_offset` spec does not name the complex -data types in its list. So the spec leaves open whether `scale_offset` -accepts them, and it does here. +complex numbers, but the `scale_offset` spec names none of the complex +types its repository defines among those it supports. """ -__all__ = ["NOT_NUMBERS", "NO_REAL_NUMBERS"] +__all__ = ["NO_REAL_NUMBERS"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index b5c10f61c0..459f953a0c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -24,7 +24,7 @@ ScaleOffsetCodecName, ScaleOffsetCodecObject, ) -from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS +from zarr_metadata.v3.codec._arithmetic import NO_REAL_NUMBERS def _rules( @@ -46,12 +46,15 @@ def _rules( def _chunk_rules( configuration: ScaleOffsetCodecConfiguration, nested: Nested, chunk: Chunk ) -> Iterator[ValidationProblem]: - """Check that the input data type supports arithmetic, and that `offset` and `scale` are values of that type. + """Check that the input data type is an integer or floating-point type, and that `offset` and `scale` are values of that type. The codec "is only defined for data types where multiplication, - division, addition, and subtraction are well-defined" + division, addition, and subtraction are well-defined", and of the data + types its repository defines it supports the integer, unsigned integer + and floating-point types (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9), - and each of `offset` and `scale` "MUST be encoded to JSON using the + so a data type that declares complex or non-numeric values is refused. + Each of `offset` and `scale` "MUST be encoded to JSON using the Zarr V3 fill value encoding for the input array's data type" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43). The rules, not the chunk rules, reject a null. @@ -59,10 +62,10 @@ def _chunk_rules( source = chunk.data_type if not isinstance(source, AcceptedDeclaration): return - if source.definition.number_class in NOT_NUMBERS: + if source.definition.number_class in NO_REAL_NUMBERS: yield ValidationProblem( (), - f"expected a chunk of a data type with arithmetic, got {shown(source.name)}", + f"expected a chunk of an integer or floating-point data type, got {shown(source.name)}", "invalid_value", ) return diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 9a107fd521..ea33f7463b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -29,13 +29,19 @@ fill_value_default=ConstantFillValue("NaT"), fill_value_canonical=numpy_time_fill_value_canonical, storage=numpy_time_storage, - number_class="integral", + number_class="non_numeric", ) """The `numpy.datetime64` data type. -Its number class is integral, since it "is compatible with any codec that -supports arrays of signed 64-bit integers" -(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). +Its number class is non-numeric: its values are moments in time, and +neither `cast_value` nor `scale_offset` names it among "the following data +types defined in this repository" that it supports +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L5-L9, +https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9). +Its own spec says it "is compatible with any codec that supports arrays of +signed 64-bit integers" +(https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120), +which those two codecs' specs do not bear out. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 90e8c628e8..4aaa3210d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -27,13 +27,13 @@ fill_value_default=ConstantFillValue("NaT"), fill_value_canonical=numpy_time_fill_value_canonical, storage=numpy_time_storage, - number_class="integral", + number_class="non_numeric", ) """The `numpy.timedelta64` data type. -Its number class is integral: like `numpy.datetime64`, it is stored as -signed 64-bit integers, and is compatible with any codec that supports -arrays of those. +Its number class is non-numeric, as that of `numpy.datetime64` is: its +values are durations, and neither `cast_value` nor `scale_offset` names it +among the data types it supports. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index 6d08e99e95..15664da244 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -57,9 +57,10 @@ A data type also declares what kind of numbers its values are with `number_class`, a `NumberClass`: integral, floating-point, complex, or not numbers. A codec that is defined only for some kinds of numbers -reads it: `cast_value` accepts only data types that model real numbers, -and `scale_offset` only those with arithmetic. A data type that does not -declare it leaves its number class unknown, and those codecs accept it. +reads it: `cast_value` and `scale_offset` refuse a data type that +declares complex or non-numeric values, as the numpy time types do. A +data type that does not declare it leaves its number class unknown, and +those codecs accept it. A chunk grid declares which array shapes it fits with `shape_rules`, a function that yields the spec violations in a grid with its diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index b947741afe..8ef6023265 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -22,6 +22,7 @@ import pytest from hypothesis import event, find, given, settings from hypothesis import strategies as st +from typing_extensions import TypedDict from tests.testing._shares import share_of from zarr_metadata import UNSET @@ -523,6 +524,18 @@ def test_fill_values_of_a_field_no_definition_reads_is_a_lookup_error() -> None: fill_values(STRUCT_DATA_TYPE, configuration, strategies=without_float32) +class _AcmeIntConfiguration(TypedDict, closed=True): + """The configuration of `acme.int`: the member a refused draw leaves out.""" + + bits: int + + +_ACME_INT = DataTypeDefinition( + name="acme.int", configuration=_AcmeIntConfiguration, number_class="integral" +) +"""An integer type with a member it requires, which `cast_value` casts to.""" + + def test_a_refused_nested_draw_is_a_value_error() -> None: """A field a configuration holds is read as it is drawn, so a strategy that draws a refused one is caught there.""" refused = DataTypeStrategy( @@ -531,7 +544,7 @@ def test_a_refused_nested_draw_is_a_value_error() -> None: ) strategies: Strategies = { CAST_VALUE_CODEC: DEFAULT_STRATEGIES[CAST_VALUE_CODEC], - NUMPY_DATETIME64_DATA_TYPE: refused, + _ACME_INT: refused, } with pytest.raises(ValueError, match="read in the strategies. scope, is refused"): find( @@ -626,56 +639,25 @@ def test_scale_offset_reaches_every_spelling_of_a_float( ) -@pytest.mark.parametrize("definition", [COMPLEX64_DATA_TYPE, COMPLEX128_DATA_TYPE], ids=_id) -@pytest.mark.parametrize("member", ["offset", "scale"]) -@pytest.mark.parametrize("part", [0, 1]) -@pytest.mark.parametrize("value", ["NaN", "Infinity", "-Infinity"]) -def test_scale_offset_reaches_a_non_finite_part_of_a_complex( - definition: DataTypeDefinition[Any], member: str, part: int, value: str -) -> None: - """A complex number's `offset` and `scale` are pairs of float fill values, either of them non-finite.""" - chunk = Chunk(None, _read(definition, definition.name)) - configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) - assert configurations is not None - find( - configurations, - lambda configuration: ( - member in configuration - and cast("tuple[object, object]", configuration[member])[part] == value - ), - settings=_REACH, - ) - - def _is_zero_spelled(value: object, width: int) -> bool: """Whether a float fill value is zero of either sign, by its value alone.""" bits = _bits(value) return value == 0 if bits is None else bits in (0, 2 ** (width - 1)) -@pytest.mark.parametrize( - ("definition", "width"), - [ - *_FLOAT_BITS.items(), - (COMPLEX64_DATA_TYPE, 32), - (COMPLEX128_DATA_TYPE, 64), - ], - ids=str, -) +@pytest.mark.parametrize(("definition", "width"), list(_FLOAT_BITS.items()), ids=str) @given(data=st.data()) def test_scale_offset_scale_is_never_zero( definition: DataTypeDefinition[Any], width: int, data: st.DataObject ) -> None: - """No spelling of a float, nor of both parts of a complex number, draws a scale of zero: decoding divides by it.""" + """No spelling of a float draws a scale of zero: decoding divides by it.""" chunk = Chunk(None, _read(definition, definition.name)) configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) assert configurations is not None configuration = data.draw(configurations) assert list(SCALE_OFFSET_CODEC.chunk_rules(configuration, {}, chunk)) == [] if "scale" in configuration: - scale = configuration["scale"] - parts = scale if isinstance(scale, tuple) else (scale,) - assert not all(_is_zero_spelled(part, width) for part in parts) + assert not _is_zero_spelled(configuration["scale"], width) def _entries(dimension: object) -> tuple[object, ...]: @@ -940,11 +922,9 @@ def _read_data_type(field: object) -> Any: # noqa: ANN401 in { *(f"{sign}int{bits}" for sign in ("", "u") for bits in (8, 16, 32, 64)), *(f"float{bits}" for bits in (16, 32, 64)), - "numpy.datetime64", - "numpy.timedelta64", } ] -"""The data types `cast_value` casts to: integers and floats, and the numpy times, which are signed 64-bit integers.""" +"""The data types `cast_value` casts to: integers and floats.""" @pytest.mark.parametrize("definition", _REAL_NUMBERS, ids=_id) diff --git a/packages/zarr-metadata/tests/v3/test_number_class.py b/packages/zarr-metadata/tests/v3/test_number_class.py index 2d35b35e62..6eb3ba4e1a 100644 --- a/packages/zarr-metadata/tests/v3/test_number_class.py +++ b/packages/zarr-metadata/tests/v3/test_number_class.py @@ -2,7 +2,7 @@ `cast_value` accepts only data types that model real numbers, on either side of the cast, and wraps only to an integral one. `scale_offset` -accepts only data types with arithmetic. Each reads the number class that +accepts the same data types. Each reads the number class that the data type's definition declares, and accepts a data type that declares none, since nothing is known of its values. """ @@ -39,8 +39,6 @@ "uint16", "uint32", "uint64", - "numpy.datetime64", - "numpy.timedelta64", ], "integral", ), @@ -52,6 +50,8 @@ "bytes", "fixed_length_utf32", "null_terminated_bytes", + "numpy.datetime64", + "numpy.timedelta64", "r*", "raw_bytes", "string", @@ -97,7 +97,7 @@ def _chunk(data_type: str) -> Chunk: _NO_REALS = 'expected a chunk of a data type that models real numbers, got "{}"' _NO_REAL_TARGET = 'expected a data type that models real numbers, got "{}"' -_NO_ARITHMETIC = 'expected a chunk of a data type with arithmetic, got "{}"' +_NO_SCALING = 'expected a chunk of an integer or floating-point data type, got "{}"' _NO_WRAP = 'expected an integral data_type to wrap to, got "{}"' @@ -132,12 +132,12 @@ def _chunk(data_type: str) -> Chunk: "int8", [((0, "configuration", "out_of_range"), _NO_WRAP.format("acme.floating_point"))], ), - # scale_offset takes every kind of number. + # scale_offset takes the same data types. ("scale_offset", "acme.integral", []), ("scale_offset", "acme.floating_point", []), - ("scale_offset", "acme.complex", []), ("scale_offset", "acme.unknown", []), - ("scale_offset", "acme.non_numeric", [((0,), _NO_ARITHMETIC.format("acme.non_numeric"))]), + ("scale_offset", "acme.complex", [((0,), _NO_SCALING.format("acme.complex"))]), + ("scale_offset", "acme.non_numeric", [((0,), _NO_SCALING.format("acme.non_numeric"))]), ], ) def test_the_arithmetic_codecs_read_the_number_class( diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index cfde42f277..7fcab1c1d2 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -189,20 +189,6 @@ def _lengths(*axes: set[int]) -> tuple[frozenset[int], ...]: [CHUNK, Chunk(), Chunk(None, _of("float32").data_type)], ), ([_scale(offset=1), "acme.vlen"], Chunk(), [Chunk(), Chunk()]), - # Where the specs disagree or say nothing, a data type is not checked. - # The numpy time types say they accept any codec of 64-bit integers, - # and scale_offset does not say whether it accepts complex numbers. - ( - [_cast("int64"), LITTLE], - _of({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}), - [ - _of( - {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}} - ), - _of("int64"), - ], - ), - ([_scale(offset=[1, 0]), LITTLE], _of("complex64"), [_of("complex64"), _of("complex64")]), ], ) def test_every_pipeline_hands_each_codec_the_chunk_the_one_before_hands_on( @@ -325,8 +311,21 @@ def test_error_a_scalar_the_cast_maps_from_that_is_not_of_the_data_type_it_is_ha @pytest.mark.parametrize("codec", [_cast("float32"), _scale()]) -@pytest.mark.parametrize("data_type", ["bool", "string", _struct("int8")]) -def test_error_an_arithmetic_codec_handed_values_that_are_no_numbers( +@pytest.mark.parametrize( + "data_type", + [ + "bool", + "string", + _struct("int8"), + # Each codec's spec lists the integer and floating-point types among + # the data types of its repository that it supports, which leaves out + # the complex types and the numpy time types. + "complex64", + {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}, + {"name": "numpy.timedelta64", "configuration": {"unit": "s", "scale_factor": 1}}, + ], +) +def test_error_an_arithmetic_codec_handed_a_data_type_its_spec_does_not_list( codec: JSONValue, data_type: JSONValue ) -> None: # The codec comes before an array -> bytes codec that nothing in scope @@ -343,10 +342,6 @@ def test_error_a_codec_handed_on_what_another_could_not_take_is_judged_too() -> ] -def test_error_a_cast_handed_complex_numbers() -> None: - assert _problems([_cast("float32"), "acme.vlen"], _of("complex64")) == [((0,), "invalid_value")] - - @pytest.mark.parametrize( ("members", "data_type", "found"), [ From 199ceb595098d81c19c6a06354712fc55963357f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 12 Nov 2026 19:13:22 +0100 Subject: [PATCH 07/11] docs(zarr-metadata): number the codec kind and number class fragments after fork PR 445 The fragments take the number of the fork PR for now, and are renumbered to the upstream PR that merges them. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/{XXXX.bugfix.md => 445.bugfix.md} | 0 .../zarr-metadata/changes/{XXXX.feature.md => 445.feature.md} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{XXXX.bugfix.md => 445.bugfix.md} (100%) rename packages/zarr-metadata/changes/{XXXX.feature.md => 445.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/XXXX.bugfix.md b/packages/zarr-metadata/changes/445.bugfix.md similarity index 100% rename from packages/zarr-metadata/changes/XXXX.bugfix.md rename to packages/zarr-metadata/changes/445.bugfix.md diff --git a/packages/zarr-metadata/changes/XXXX.feature.md b/packages/zarr-metadata/changes/445.feature.md similarity index 100% rename from packages/zarr-metadata/changes/XXXX.feature.md rename to packages/zarr-metadata/changes/445.feature.md From 84a4992d24a1bf39a24184d49d57fae58674e95e Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 12 Nov 2026 19:41:55 +0100 Subject: [PATCH 08/11] build(deps): require packaging>=24.0 and drop pytest-accept (#4626) * chore(deps-dev): bump the python-dependencies group across 1 directory with 13 updates Bumps the python-dependencies group with 13 updates in the / directory: | Package | From | To | | --- | --- | --- | | [obstore](https://github.com/geospatial-jeff/pyasyncio-benchmark) | `0.12.0` | `0.12.1` | | [coverage](https://github.com/coveragepy/coveragepy) | `7.16.2` | `7.17.2` | | [pytest](https://github.com/pytest-dev/pytest) | `9.1.1` | `9.2.0` | | [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) | `1.4.0` | `1.4.1.post1` | | [hypothesis](https://github.com/HypothesisWorks/hypothesis) | `6.170.5` | `6.170.14` | | [pytest-benchmark](https://github.com/ionelmc/pytest-benchmark) | `5.3.0` | `5.3.1` | | [pytest-codspeed](https://github.com/CodSpeedHQ/pytest-codspeed) | `5.0.3` | `5.0.4` | | [uv](https://github.com/astral-sh/uv) | `0.13.5` | `0.13.7` | | [moto[s3,server]](https://github.com/getmoto/moto) | `5.2.4` | `5.2.5` | | [mkdocs-material[imaging]](https://github.com/squidfunk/mkdocs-material) | `9.7.7` | `9.8.0` | | [markdown-exec[ansi]](https://github.com/pawamoy/markdown-exec) | `1.12.5` | `1.12.6` | | [ruff](https://github.com/astral-sh/ruff) | `0.17.1` | `0.17.3` | | [mypy](https://github.com/python/mypy) | `2.4.0` | `2.4.1` | Updates `obstore` from 0.12.0 to 0.12.1 - [Commits](https://github.com/geospatial-jeff/pyasyncio-benchmark/commits) Updates `coverage` from 7.16.2 to 7.17.2 - [Release notes](https://github.com/coveragepy/coveragepy/releases) - [Changelog](https://github.com/coveragepy/coveragepy/blob/main/CHANGES.rst) - [Commits](https://github.com/coveragepy/coveragepy/compare/7.16.2...7.17.2) Updates `pytest` from 9.1.1 to 9.2.0 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/9.1.1...9.2.0) Updates `pytest-asyncio` from 1.4.0 to 1.4.1.post1 - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v1.4.0...v1.4.1.post1) Updates `hypothesis` from 6.170.5 to 6.170.14 - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/v6.170.5...v6.170.14) Updates `pytest-benchmark` from 5.3.0 to 5.3.1 - [Release notes](https://github.com/ionelmc/pytest-benchmark/releases) - [Changelog](https://github.com/ionelmc/pytest-benchmark/blob/master/CHANGELOG.rst) - [Commits](https://github.com/ionelmc/pytest-benchmark/compare/v5.3.0...v5.3.1) Updates `pytest-codspeed` from 5.0.3 to 5.0.4 - [Release notes](https://github.com/CodSpeedHQ/pytest-codspeed/releases) - [Changelog](https://github.com/CodSpeedHQ/pytest-codspeed/blob/master/CHANGELOG.md) - [Commits](https://github.com/CodSpeedHQ/pytest-codspeed/compare/v5.0.3...v5.0.4) Updates `uv` from 0.13.5 to 0.13.7 - [Release notes](https://github.com/astral-sh/uv/releases) - [Changelog](https://github.com/astral-sh/uv/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/uv/compare/0.13.5...0.13.7) Updates `moto[s3,server]` from 5.2.4 to 5.2.5 - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/5.2.4...5.2.5) Updates `mkdocs-material[imaging]` from 9.7.7 to 9.8.0 - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.7.7...9.8.0) Updates `markdown-exec[ansi]` from 1.12.5 to 1.12.6 - [Release notes](https://github.com/pawamoy/markdown-exec/releases) - [Changelog](https://github.com/pawamoy/markdown-exec/blob/main/CHANGELOG.md) - [Commits](https://github.com/pawamoy/markdown-exec/compare/1.12.5...1.12.6) Updates `ruff` from 0.17.1 to 0.17.3 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.17.1...0.17.3) Updates `mypy` from 2.4.0 to 2.4.1 - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v2.4.0...v2.4.1) --- updated-dependencies: - dependency-name: obstore dependency-version: 0.12.1 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: coverage dependency-version: 7.17.2 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-dependencies - dependency-name: pytest dependency-version: 9.2.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-dependencies - dependency-name: pytest-asyncio dependency-version: 1.4.1.post1 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: hypothesis dependency-version: 6.170.14 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: pytest-benchmark dependency-version: 5.3.1 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: pytest-codspeed dependency-version: 5.0.4 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: uv dependency-version: 0.13.7 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: moto[s3,server] dependency-version: 5.2.5 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: mkdocs-material[imaging] dependency-version: 9.8.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-dependencies - dependency-name: markdown-exec[ansi] dependency-version: 1.12.6 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: ruff dependency-version: 0.17.3 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: mypy dependency-version: 2.4.1 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies ... Signed-off-by: dependabot[bot] * test: keep the suite running with pytest 9.2 and hypothesis 6.170.7+ pytest 9.2 applies the configured filterwarnings while importing plugins (pytest#12697), so the "error" filter turns the ast.Num DeprecationWarning that astor (a dependency of pytest-accept) triggers at import on Python < 3.14 into a startup failure (exit code 3). Ignore that one astor-scoped deprecation, which pytest <= 9.1 never routed through the filters. pytest 9.2 also requires packaging>=24, which conflicts with the min_deps pin packaging==22.* (the declared runtime floor packaging>=22.0). Keep that floor tested by overriding pytest to <9.2 in the min_deps env only, via UV_OVERRIDE and ci/min_deps-overrides.txt; every other env uses pytest 9.2.0. hypothesis 6.170.7 annotates run_state_machine_as_test as taking type[RuleBasedStateMachine] (its docstring still accepts any zero-arg factory), so mypy now reports arg-type instead of no-untyped-call at the four call sites that pass a factory function. Update the ignore codes; no test logic changes. * test: drop the unused pytest-accept plugin pytest-accept was added in #2589 for a `hatch run doctest:fix` script that ran `pytest --accept` over the RST doctests. That script and those doctests are gone, and nothing in the justfile or CI passes `--accept`. The plugin's only remaining effect was importing astor (last released in 2019) at every pytest startup, whose `ast.Num` lookup pytest 9.2 turns into a startup error under our "error" warning filter. Removing the plugin removes astor, so the astor-scoped filter added for pytest 9.2 is no longer needed. Assisted-by: ClaudeCode:claude-opus-5-5 * build(deps): require packaging>=24.0 pytest 9.2 requires packaging>=24, so the min_deps env could no longer pin packaging==22.*. Instead of holding min_deps on pytest<9.2 through a UV_OVERRIDE file, raise the runtime floor to packaging>=24.0 (released March 2024) and test that floor in min_deps. zarr uses only packaging.version.Version and parse, which behave the same on 22 and 24. min_deps now runs the same pytest as every other env, and the override file and env var are gone. Assisted-by: ClaudeCode:claude-opus-5-5 * docs: add a changelog entry for the packaging>=24.0 floor Assisted-by: ClaudeCode:claude-opus-5-5 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: invisiblefix-bot <355452247+invisiblefix-bot@users.noreply.github.com> --- changes/4626.removal.md | 1 + ci/big-endian/Dockerfile | 2 +- ci/min_deps-overrides.txt | 3 --- docs/user-guide/installation.md | 2 +- pyproject.toml | 13 ++----------- uv.lock | 30 +----------------------------- 6 files changed, 6 insertions(+), 45 deletions(-) create mode 100644 changes/4626.removal.md delete mode 100644 ci/min_deps-overrides.txt diff --git a/changes/4626.removal.md b/changes/4626.removal.md new file mode 100644 index 0000000000..34c208c0ba --- /dev/null +++ b/changes/4626.removal.md @@ -0,0 +1 @@ +The minimum supported packaging version is now 24.0. diff --git a/ci/big-endian/Dockerfile b/ci/big-endian/Dockerfile index b473c47e43..a2700103ba 100644 --- a/ci/big-endian/Dockerfile +++ b/ci/big-endian/Dockerfile @@ -32,7 +32,7 @@ RUN git clone -q --depth 1 --branch 1.1.2 https://github.com/google/crc32c /tmp/ RUN python3 -m venv --system-site-packages /venv ENV PATH=/venv/bin:$PATH -RUN pip install -q --no-binary google-crc32c google-crc32c msgspec pytest-accept +RUN pip install -q --no-binary google-crc32c google-crc32c msgspec # Debian sid's python3-dateutil 3.0.0 declares a dependency on the `tzdata` distribution, # which no Debian package installs, so `pip check` fails on the image as built. tzdata is # pure Python; install it from PyPI. diff --git a/ci/min_deps-overrides.txt b/ci/min_deps-overrides.txt deleted file mode 100644 index 2eb5ec49de..0000000000 --- a/ci/min_deps-overrides.txt +++ /dev/null @@ -1,3 +0,0 @@ -# pytest>=9.2 requires packaging>=24, but min_deps must keep testing the -# declared runtime floor packaging>=22.0, so this env stays on pytest<9.2. -pytest<9.2 diff --git a/docs/user-guide/installation.md b/docs/user-guide/installation.md index a7487e83c8..4ae4baa76a 100644 --- a/docs/user-guide/installation.md +++ b/docs/user-guide/installation.md @@ -5,7 +5,7 @@ Required dependencies include: - [Python](https://docs.python.org/3/) (3.12 or later) -- [packaging](https://packaging.pypa.io/en/stable/) (22.0 or later) +- [packaging](https://packaging.pypa.io/en/stable/) (24.0 or later) - [numpy](https://numpy.org) (2.0 or later) - [numcodecs](https://numcodecs.readthedocs.io/en/stable/) (0.14 or later) - [google-crc32c](https://github.com/googleapis/python-crc32c) (1.5 or later) diff --git a/pyproject.toml b/pyproject.toml index 983ef58c96..839b5fdef8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ maintainers = [ ] requires-python = ">=3.12" dependencies = [ - 'packaging>=22.0', + 'packaging>=24.0', 'numpy>=2', # 0.16.6 and 0.17.1 refuse to unpickle untrusted data (GHSA-jqc8-qjfv-5m6r); 0.17.0 does not. 'numcodecs>=0.16.6,!=0.17.0', @@ -115,7 +115,6 @@ test = [ "pytest==9.2.0", "pytest-asyncio==1.4.1.post1", "pytest-cov==7.1.0", - "pytest-accept==0.3.0", "numpydoc==1.11.0", "hypothesis==6.170.14", "pytest-reportlog==1.0.0", @@ -251,7 +250,7 @@ python = "3.12" features = ["remote"] dependency-groups = ["remote-tests"] extra-dependencies = [ - 'packaging==22.*', + 'packaging==24.*', 'numpy==2.0.*', 'numcodecs==0.16.*', 'fsspec==2023.10.0', @@ -263,10 +262,6 @@ extra-dependencies = [ 'msgspec==0.19.*', ] -[tool.hatch.envs.min_deps.env-vars] -# Keep pytest<9.2 here so packaging==22.* (our declared floor) still resolves. -UV_OVERRIDE = "{root}/ci/min_deps-overrides.txt" - [tool.hatch.envs.default] installer = "uv" @@ -462,10 +457,6 @@ filterwarnings = [ # loop-lifecycle changes. "ignore:Exception ignored in[\\s\\S]*=9.2 applies these filters while importing plugins (pytest#12697). pytest-accept - # imports astor, which looks up the deprecated ast.Num at import time on Python < 3.14, - # so "error" above would abort the run before collection. - "ignore:ast\\.\\w+ is deprecated:DeprecationWarning:astor", ] markers = [ "gpu: mark a test as requiring CuPy and GPU", diff --git a/uv.lock b/uv.lock index 38e8220a06..8dbf06aab4 100644 --- a/uv.lock +++ b/uv.lock @@ -297,15 +297,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/25/4235f4b375a67a37ac595bf478a5407a744eaf7b64113c1c3a5432b8d954/ast_serialize-0.13.1-cp39-abi3-win_arm64.whl", hash = "sha256:48f11d96c9ec71119612f8e79a8c55c12f2b136716d68347433c561d27a25f43", size = 1139035, upload-time = "2026-10-17T00:04:15.767Z" }, ] -[[package]] -name = "astor" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, -] - [[package]] name = "astroid" version = "4.3.4" @@ -2837,19 +2828,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl", hash = "sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588", size = 411776, upload-time = "2026-10-27T21:30:19.131Z" }, ] -[[package]] -name = "pytest-accept" -version = "0.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astor" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/42/573e6dcd8d1a5fc779d525268d5036f710d018cdeefbae2bd5559687a6d5/pytest_accept-0.3.0.tar.gz", hash = "sha256:6f4e03e2492621e10b7678a331828274498cca04686b15b0f3e704e16e651068", size = 29744, upload-time = "2026-06-11T18:44:39.03Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/40/79f6f13e616f580b9c35110376818c76cacc54b70d2965bf1f82be336c66/pytest_accept-0.3.0-py3-none-any.whl", hash = "sha256:5e73f27020853b861a63dc630dfe63124af4bd60e0e0bb692e1376b99798f2b5", size = 39846, upload-time = "2026-06-11T18:44:37.965Z" }, -] - [[package]] name = "pytest-asyncio" version = "1.4.1.post1" @@ -3872,7 +3850,6 @@ dev = [ { name = "numpydoc" }, { name = "obstore" }, { name = "pytest" }, - { name = "pytest-accept" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, { name = "pytest-codspeed" }, @@ -3914,7 +3891,6 @@ remote-tests = [ { name = "numpydoc" }, { name = "obstore" }, { name = "pytest" }, - { name = "pytest-accept" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, { name = "pytest-codspeed" }, @@ -3930,7 +3906,6 @@ test = [ { name = "hypothesis" }, { name = "numpydoc" }, { name = "pytest" }, - { name = "pytest-accept" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, { name = "pytest-codspeed" }, @@ -3951,7 +3926,7 @@ requires-dist = [ { name = "numcodecs", specifier = ">=0.16.6,!=0.17.0" }, { name = "numpy", specifier = ">=2" }, { name = "obstore", marker = "extra == 'remote'", specifier = ">=0.5.1" }, - { name = "packaging", specifier = ">=22.0" }, + { name = "packaging", specifier = ">=24.0" }, { name = "typer", marker = "extra == 'cli'" }, { name = "typing-extensions", specifier = ">=4.14" }, { name = "universal-pathlib", marker = "extra == 'optional'" }, @@ -3979,7 +3954,6 @@ dev = [ { name = "numpydoc", specifier = "==1.11.0" }, { name = "obstore", specifier = ">=0.5.1" }, { name = "pytest", specifier = "==9.2.0" }, - { name = "pytest-accept", specifier = "==0.3.0" }, { name = "pytest-asyncio", specifier = "==1.4.1.post1" }, { name = "pytest-benchmark", specifier = "==5.3.1" }, { name = "pytest-codspeed", specifier = "==5.0.4" }, @@ -4019,7 +3993,6 @@ remote-tests = [ { name = "numpydoc", specifier = "==1.11.0" }, { name = "obstore", specifier = ">=0.5.1" }, { name = "pytest", specifier = "==9.2.0" }, - { name = "pytest-accept", specifier = "==0.3.0" }, { name = "pytest-asyncio", specifier = "==1.4.1.post1" }, { name = "pytest-benchmark", specifier = "==5.3.1" }, { name = "pytest-codspeed", specifier = "==5.0.4" }, @@ -4035,7 +4008,6 @@ test = [ { name = "hypothesis", specifier = "==6.170.14" }, { name = "numpydoc", specifier = "==1.11.0" }, { name = "pytest", specifier = "==9.2.0" }, - { name = "pytest-accept", specifier = "==0.3.0" }, { name = "pytest-asyncio", specifier = "==1.4.1.post1" }, { name = "pytest-benchmark", specifier = "==5.3.1" }, { name = "pytest-codspeed", specifier = "==5.0.4" }, From c681a1a90391b3df9a8db862ae365f96e2502fad Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 12 Nov 2026 19:42:07 +0100 Subject: [PATCH 09/11] docs(zarr-metadata): number the codec kind and number class fragments after #4628 Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/changes/{445.bugfix.md => 4628.bugfix.md} | 0 .../zarr-metadata/changes/{445.feature.md => 4628.feature.md} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{445.bugfix.md => 4628.bugfix.md} (100%) rename packages/zarr-metadata/changes/{445.feature.md => 4628.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/445.bugfix.md b/packages/zarr-metadata/changes/4628.bugfix.md similarity index 100% rename from packages/zarr-metadata/changes/445.bugfix.md rename to packages/zarr-metadata/changes/4628.bugfix.md diff --git a/packages/zarr-metadata/changes/445.feature.md b/packages/zarr-metadata/changes/4628.feature.md similarity index 100% rename from packages/zarr-metadata/changes/445.feature.md rename to packages/zarr-metadata/changes/4628.feature.md From e8e8d961440b27fefddffbab519f11a677c17da2 Mon Sep 17 00:00:00 2001 From: ZIHAN DAI <99155080+PDGGK@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:07:55 +1100 Subject: [PATCH 10/11] fix: migrate v2 arrays whose fill_value is null (#4616) * fix: migrate v2 arrays whose fill_value is null Zarr v2 allows "fill_value": null, and xarray writes it for variables without a _FillValue. Zarr v3 requires a fill value, so `zarr migrate v3` raised a TypeError on such arrays. Use the data type's default scalar, which is what zarr-python already returns for unwritten chunks of the v2 array. * docs: add changelog entry for #4616 * test(migrate): cover a partly written chunk in the null fill value test Write only part of the first chunk, so the test checks that the padding stored in a partial chunk reads back the same after migration, and compare the v2 read against the same expected values as the v3 read. Assisted-by: ClaudeCode:claude-opus-5-5 --------- Co-authored-by: Davis Vann Bennett --- changes/4616.bugfix.md | 5 +++++ src/zarr/metadata/migrate_v3.py | 8 ++++++- tests/test_cli/test_migrate_v3.py | 36 +++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 changes/4616.bugfix.md diff --git a/changes/4616.bugfix.md b/changes/4616.bugfix.md new file mode 100644 index 0000000000..5745d77c0f --- /dev/null +++ b/changes/4616.bugfix.md @@ -0,0 +1,5 @@ +`zarr migrate v3` now migrates Zarr format 2 arrays whose `fill_value` is +`null`, such as the integer variables and coordinates xarray writes. The v3 +metadata uses the data type's default fill value, which is what reading an +unwritten chunk of the v2 array already returned. Before, the migration failed +with `TypeError: Cannot convert object None ... to a scalar`. diff --git a/src/zarr/metadata/migrate_v3.py b/src/zarr/metadata/migrate_v3.py index ad177e19fb..cd94f12d84 100644 --- a/src/zarr/metadata/migrate_v3.py +++ b/src/zarr/metadata/migrate_v3.py @@ -209,12 +209,18 @@ def _convert_array_metadata(metadata_v2: ArrayV2Metadata) -> ArrayV3Metadata: bytes_bytes_codec = _convert_compressor(metadata_v2.compressor, metadata_v2.dtype) codecs.append(bytes_bytes_codec) + # Zarr v2 allows a null fill value, Zarr v3 does not. Use the default that is + # already used when reading chunks of the v2 array that were never written. + fill_value: object = metadata_v2.fill_value + if fill_value is None: + fill_value = metadata_v2.dtype.default_scalar() + return ArrayV3Metadata( shape=metadata_v2.shape, data_type=metadata_v2.dtype, chunk_grid=RegularChunkGridMetadata(chunk_shape=metadata_v2.chunks), chunk_key_encoding=chunk_key_encoding, - fill_value=metadata_v2.fill_value, + fill_value=fill_value, codecs=codecs, attributes=metadata_v2.attributes, dimension_names=None, diff --git a/tests/test_cli/test_migrate_v3.py b/tests/test_cli/test_migrate_v3.py index 83401487db..2707341e1d 100644 --- a/tests/test_cli/test_migrate_v3.py +++ b/tests/test_cli/test_migrate_v3.py @@ -455,6 +455,42 @@ def test_migrate_endian( assert np.all(zarr_array[:] == 1) +@pytest.mark.parametrize( + ("dtype", "expected_fill_value"), + [(" None: + """A null v2 fill value becomes the default that is used when reading the v2 array. + + The first chunk is only partly written, so its stored padding must read back + the same after migration. The second chunk is never written. + """ + zarr_array = zarr.create_array( + store=local_store, + shape=(10,), + chunks=(5,), + dtype=dtype, + zarr_format=2, + fill_value=None, + ) + zarr_array[:3] = 1 + assert zarr_array.metadata.fill_value is None + v2_values = zarr_array[:] + + result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) + assert result.exit_code == 0 + assert (local_store.root / "zarr.json").exists() + + zarr_array = zarr.open_array(local_store.root, zarr_format=3) + assert zarr_array.metadata.fill_value == expected_fill_value + expected = np.array([1] * 3 + [expected_fill_value] * 7, dtype=dtype) + assert np.array_equal(v2_values, expected) + assert np.array_equal(zarr_array[:], expected) + + @pytest.mark.parametrize("node_type", ["array", "group"]) def test_migrate_v3(local_store: LocalStore, node_type: str) -> None: """Attempting to convert a v3 array/group should always fail""" From 6e7bcfe4edeb87261e22e78ec512131a04bf7756 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 12 Nov 2026 22:59:15 +0100 Subject: [PATCH 11/11] docs(zarr-metadata): define the non-numeric class as values that are not plain numbers The open zarr-extensions proposals include an optional (nullable) data type, a fixed-size list and a ragged ndarray. Their values hold numbers without being numbers, and neither arithmetic codec's spec lists them, so they belong in `non_numeric`, which said "not numbers at all". It now says every value that is not a plain number, naming records, lists and values that may be missing. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/src/zarr_metadata/v3/_definition.py | 11 +++++++---- .../src/zarr_metadata/v3/codec/_arithmetic.py | 2 +- .../zarr-metadata/src/zarr_metadata/v3/definition.py | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index fd0b4a089b..0d734d81e1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -128,7 +128,7 @@ def unknown_storage(*_: object) -> None: NumberClass = Literal["integral", "floating_point", "complex", "non_numeric"] -"""What kind of numbers a data type's values are: integers, floating-point numbers, complex numbers, or not numbers at all. +"""What kind of numbers a data type's values are: integers, floating-point numbers, complex numbers, or values that are not plain numbers. A codec that is defined only for some kinds of numbers reads this. For example, `cast_value` "is only defined for data types that model real @@ -137,8 +137,11 @@ def unknown_storage(*_: object) -> None: and its `wrap` mode is "Only permitted when data_type is an integral type" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L84). -Values that are truth values, times, text, byte strings or records are -`non_numeric`. +`non_numeric` is every other kind of value: truth values, times, text, +byte strings, records and lists, and a value that may be missing, such +as an optional integer, whose values are an integer or nothing. A record +or a list of numbers, or an optional number, holds numbers but is not one, +so the codecs that read the number class treat it as they treat text. """ @@ -213,7 +216,7 @@ class DataTypeDefinition(WithFillValue[C], kind=True, format=3): own. `number_class` declares what kind of numbers the values are: integral, - floating-point, complex, or not numbers. A codec that is defined only + floating-point, complex, or not plain numbers. A codec that is defined only for some kinds of numbers reads it, as `cast_value` reads it to accept only data types that model real numbers. A data type that does not declare it leaves its number class unknown, and such a codec accepts diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py index d8eaa5aadc..60c473d48d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py @@ -23,7 +23,7 @@ from zarr_metadata.v3._definition import NumberClass NO_REAL_NUMBERS: Final[frozenset[NumberClass]] = frozenset({"non_numeric", "complex"}) -"""The number classes that neither codec takes: values that are not numbers, and complex numbers. +"""The number classes that neither codec takes: values that are not plain numbers, and complex numbers. Addition, subtraction, multiplication and division are well-defined on complex numbers, but the `scale_offset` spec names none of the complex diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index 15664da244..c98d6460c1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -56,7 +56,7 @@ calls it for a data type declaration the scope read. A data type also declares what kind of numbers its values are with `number_class`, a `NumberClass`: integral, floating-point, complex, or -not numbers. A codec that is defined only for some kinds of numbers +not plain numbers. A codec that is defined only for some kinds of numbers reads it: `cast_value` and `scale_offset` refuse a data type that declares complex or non-numeric values, as the numpy time types do. A data type that does not declare it leaves its number class unknown, and