atdata 0.3.0b1__py3-none-any.whl → 0.3.1b1__py3-none-any.whl

atdata/lexicons/__init__.py

@@ -0,0 +1,121 @@
+"""ATProto Lexicon definitions for the atdata federation.
+
+This package contains the canonical Lexicon JSON files for the
+``ac.foundation.dataset`` namespace. These define the ATProto record
+types used by atdata for publishing schemas, datasets, and lenses
+to the AT Protocol network.
+
+Lexicons:
+    ac.foundation.dataset.schema
+        Versioned sample type definitions (PackableSample schemas).
+    ac.foundation.dataset.record
+        Dataset index records pointing to WebDataset storage.
+    ac.foundation.dataset.lens
+        Bidirectional transformations between schemas.
+    ac.foundation.dataset.schemaType
+        Extensible token for schema format identifiers.
+    ac.foundation.dataset.arrayFormat
+        Extensible token for array serialization formats.
+    ac.foundation.dataset.storageExternal
+        External URL-based storage (S3, HTTP, IPFS).
+    ac.foundation.dataset.storageBlobs
+        ATProto PDS blob-based storage.
+    ac.foundation.dataset.getLatestSchema
+        XRPC query for fetching the latest schema version.
+
+The ``ndarray_shim.json`` file defines the standard NDArray type
+for use within JSON Schema definitions.
+
+Examples:
+    >>> from atdata.lexicons import load_lexicon
+    >>> schema_lex = load_lexicon("ac.foundation.dataset.schema")
+    >>> schema_lex["id"]
+    'ac.foundation.dataset.schema'
+"""
+
+import json
+from importlib import resources
+from functools import lru_cache
+from typing import Any
+
+
+NAMESPACE = "ac.foundation.dataset"
+
+LEXICON_IDS = (
+    f"{NAMESPACE}.schema",
+    f"{NAMESPACE}.record",
+    f"{NAMESPACE}.lens",
+    f"{NAMESPACE}.schemaType",
+    f"{NAMESPACE}.arrayFormat",
+    f"{NAMESPACE}.storageExternal",
+    f"{NAMESPACE}.storageBlobs",
+    f"{NAMESPACE}.getLatestSchema",
+)
+
+
+@lru_cache(maxsize=16)
+def load_lexicon(lexicon_id: str) -> dict[str, Any]:
+    """Load a lexicon definition by its NSID.
+
+    Args:
+        lexicon_id: The lexicon NSID, e.g. ``"ac.foundation.dataset.schema"``.
+
+    Returns:
+        Parsed JSON dictionary containing the lexicon definition.
+
+    Raises:
+        FileNotFoundError: If no lexicon file exists for the given ID.
+
+    Examples:
+        >>> lex = load_lexicon("ac.foundation.dataset.schema")
+        >>> lex["defs"]["main"]["type"]
+        'record'
+    """
+    filename = f"{lexicon_id}.json"
+    ref = resources.files(__package__).joinpath(filename)
+    try:
+        text = ref.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        raise FileNotFoundError(
+            f"No lexicon file found for '{lexicon_id}'. "
+            f"Expected {filename} in {__package__}."
+        ) from None
+    return json.loads(text)
+
+
+@lru_cache(maxsize=1)
+def load_ndarray_shim() -> dict[str, Any]:
+    """Load the NDArray JSON Schema shim definition.
+
+    Returns:
+        Parsed JSON dictionary containing the NDArray shim schema.
+
+    Examples:
+        >>> shim = load_ndarray_shim()
+        >>> shim["$defs"]["ndarray"]["type"]
+        'string'
+    """
+    ref = resources.files(__package__).joinpath("ndarray_shim.json")
+    return json.loads(ref.read_text(encoding="utf-8"))
+
+
+def list_lexicons() -> tuple[str, ...]:
+    """Return the tuple of all known lexicon NSIDs.
+
+    Returns:
+        Tuple of lexicon ID strings.
+
+    Examples:
+        >>> "ac.foundation.dataset.schema" in list_lexicons()
+        True
+    """
+    return LEXICON_IDS
+
+
+__all__ = [
+    "NAMESPACE",
+    "LEXICON_IDS",
+    "load_lexicon",
+    "load_ndarray_shim",
+    "list_lexicons",
+]

atdata/lexicons/ac.foundation.dataset.arrayFormat.json

@@ -0,0 +1,16 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.arrayFormat",
+  "defs": {
+    "main": {
+      "type": "string",
+      "description": "Array serialization format identifier for NDArray fields in sample schemas. Known values correspond to token definitions in this Lexicon. Each format has versioned specifications maintained by foundation.ac at canonical URLs.",
+      "knownValues": ["ndarrayBytes"],
+      "maxLength": 50
+    },
+    "ndarrayBytes": {
+      "type": "token",
+      "description": "Numpy .npy binary format for NDArray serialization. Stores arrays with dtype and shape in binary header. Versions maintained at https://foundation.ac/schemas/atdata-ndarray-bytes/{version}/"
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.getLatestSchema.json

@@ -0,0 +1,78 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.getLatestSchema",
+  "defs": {
+    "main": {
+      "type": "query",
+      "description": "Get the latest version of a sample schema by its permanent NSID identifier",
+      "parameters": {
+        "type": "params",
+        "required": [
+          "schemaId"
+        ],
+        "properties": {
+          "schemaId": {
+            "type": "string",
+            "description": "The permanent NSID identifier for the schema (the {NSID} part of the rkey {NSID}@{semver})",
+            "maxLength": 500
+          }
+        }
+      },
+      "output": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "required": [
+            "uri",
+            "version",
+            "record"
+          ],
+          "properties": {
+            "uri": {
+              "type": "string",
+              "description": "AT-URI of the latest schema version",
+              "maxLength": 500
+            },
+            "version": {
+              "type": "string",
+              "description": "Semantic version of the latest schema",
+              "maxLength": 20
+            },
+            "record": {
+              "type": "ref",
+              "ref": "ac.foundation.dataset.schema",
+              "description": "The full schema record"
+            },
+            "allVersions": {
+              "type": "array",
+              "description": "All available versions (optional, sorted by semver descending)",
+              "items": {
+                "type": "object",
+                "required": [
+                  "uri",
+                  "version"
+                ],
+                "properties": {
+                  "uri": {
+                    "type": "string",
+                    "maxLength": 500
+                  },
+                  "version": {
+                    "type": "string",
+                    "maxLength": 20
+                  }
+                }
+              }
+            }
+          }
+        }
+      },
+      "errors": [
+        {
+          "name": "SchemaNotFound",
+          "description": "No schema found with the given NSID"
+        }
+      ]
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.lens.json

@@ -0,0 +1,99 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.lens",
+  "defs": {
+    "main": {
+      "type": "record",
+      "description": "Bidirectional transformation (Lens) between two sample types, with code stored in external repositories",
+      "key": "tid",
+      "record": {
+        "type": "object",
+        "required": [
+          "name",
+          "sourceSchema",
+          "targetSchema",
+          "getterCode",
+          "putterCode",
+          "createdAt"
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "Human-readable lens name",
+            "maxLength": 100
+          },
+          "sourceSchema": {
+            "type": "string",
+            "description": "AT-URI reference to source schema",
+            "maxLength": 500
+          },
+          "targetSchema": {
+            "type": "string",
+            "description": "AT-URI reference to target schema",
+            "maxLength": 500
+          },
+          "description": {
+            "type": "string",
+            "description": "What this transformation does",
+            "maxLength": 1000
+          },
+          "getterCode": {
+            "type": "ref",
+            "ref": "#codeReference",
+            "description": "Code reference for getter function (Source -> Target)"
+          },
+          "putterCode": {
+            "type": "ref",
+            "ref": "#codeReference",
+            "description": "Code reference for putter function (Target, Source -> Source)"
+          },
+          "language": {
+            "type": "string",
+            "description": "Programming language of the lens implementation (e.g., 'python', 'typescript')",
+            "maxLength": 50
+          },
+          "metadata": {
+            "type": "object",
+            "description": "Arbitrary metadata (author, performance notes, etc.)"
+          },
+          "createdAt": {
+            "type": "string",
+            "format": "datetime",
+            "description": "Timestamp when this lens was created"
+          }
+        }
+      }
+    },
+    "codeReference": {
+      "type": "object",
+      "description": "Reference to code in an external repository (GitHub, tangled.org, etc.)",
+      "required": [
+        "repository",
+        "commit",
+        "path"
+      ],
+      "properties": {
+        "repository": {
+          "type": "string",
+          "description": "Repository URL (e.g., 'https://github.com/user/repo' or 'at://did/tangled.repo/...')",
+          "maxLength": 500
+        },
+        "commit": {
+          "type": "string",
+          "description": "Git commit hash (ensures immutability)",
+          "maxLength": 40
+        },
+        "path": {
+          "type": "string",
+          "description": "Path to function within repository (e.g., 'lenses/vision.py:rgb_to_grayscale')",
+          "maxLength": 500
+        },
+        "branch": {
+          "type": "string",
+          "description": "Optional branch name (for reference, commit hash is authoritative)",
+          "maxLength": 100
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.record.json

@@ -0,0 +1,96 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.record",
+  "defs": {
+    "main": {
+      "type": "record",
+      "description": "Index record for a WebDataset-backed dataset with references to storage location and sample schema",
+      "key": "tid",
+      "record": {
+        "type": "object",
+        "required": [
+          "name",
+          "schemaRef",
+          "storage",
+          "createdAt"
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "Human-readable dataset name",
+            "maxLength": 200
+          },
+          "schemaRef": {
+            "type": "string",
+            "format": "at-uri",
+            "description": "AT-URI reference to the schema record for this dataset's samples",
+            "maxLength": 500
+          },
+          "storage": {
+            "type": "union",
+            "description": "Storage location for dataset files (WebDataset tar archives)",
+            "refs": [
+              "ac.foundation.dataset.storageExternal",
+              "ac.foundation.dataset.storageBlobs"
+            ]
+          },
+          "description": {
+            "type": "string",
+            "description": "Human-readable description of the dataset",
+            "maxLength": 5000
+          },
+          "metadata": {
+            "type": "bytes",
+            "description": "Msgpack-encoded metadata dict for arbitrary extended key-value pairs. Use this for additional metadata beyond the core top-level fields (license, tags, size). Top-level fields are preferred for discoverable/searchable metadata.",
+            "maxLength": 100000
+          },
+          "tags": {
+            "type": "array",
+            "description": "Searchable tags for dataset discovery. Aligns with Schema.org keywords property.",
+            "items": {
+              "type": "string",
+              "maxLength": 150
+            },
+            "maxLength": 30
+          },
+          "size": {
+            "type": "ref",
+            "ref": "#datasetSize",
+            "description": "Dataset size information (optional)"
+          },
+          "license": {
+            "type": "string",
+            "description": "License identifier or URL. SPDX identifiers recommended (e.g., MIT, Apache-2.0, CC-BY-4.0) or full SPDX URLs (e.g., http://spdx.org/licenses/MIT). Aligns with Schema.org license property.",
+            "maxLength": 200
+          },
+          "createdAt": {
+            "type": "string",
+            "format": "datetime",
+            "description": "Timestamp when this dataset record was created"
+          }
+        }
+      }
+    },
+    "datasetSize": {
+      "type": "object",
+      "description": "Information about dataset size",
+      "properties": {
+        "samples": {
+          "type": "integer",
+          "description": "Total number of samples in the dataset",
+          "minimum": 0
+        },
+        "bytes": {
+          "type": "integer",
+          "description": "Total size in bytes",
+          "minimum": 0
+        },
+        "shards": {
+          "type": "integer",
+          "description": "Number of WebDataset shards",
+          "minimum": 1
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.schema.json

@@ -0,0 +1,107 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.schema",
+  "defs": {
+    "main": {
+      "type": "record",
+      "description": "Definition of a PackableSample-compatible sample type. Supports versioning via rkey format: {NSID}@{semver}. Schema format is extensible via union type.",
+      "key": "any",
+      "record": {
+        "type": "object",
+        "required": [
+          "name",
+          "version",
+          "schemaType",
+          "schema",
+          "createdAt"
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "Human-readable display name for this sample type. Used for documentation and UI. The NSID in the record URI provides unique identification; name collisions across NSIDs are acceptable.",
+            "maxLength": 100
+          },
+          "version": {
+            "type": "string",
+            "description": "Semantic version (e.g., '1.0.0')",
+            "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$",
+            "maxLength": 100
+          },
+          "schemaType": {
+            "type": "ref",
+            "ref": "ac.foundation.dataset.schemaType",
+            "description": "Type of schema definition. This field indicates which union member is present in the schema field."
+          },
+          "schema": {
+            "type": "union",
+            "refs": ["ac.foundation.dataset.schema#jsonSchemaFormat"],
+            "closed": false,
+            "description": "Schema definition for this sample type. Currently supports JSON Schema Draft 7. Union allows for future schema formats (Avro, Protobuf, etc.) without breaking changes."
+          },
+          "description": {
+            "type": "string",
+            "description": "Human-readable description of what this sample type represents",
+            "maxLength": 5000
+          },
+          "metadata": {
+            "type": "object",
+            "description": "Optional metadata about this schema. Common fields include license and tags, but any additional fields are permitted.",
+            "maxProperties": 50,
+            "properties": {
+              "license": {
+                "type": "string",
+                "description": "License identifier or URL. SPDX identifiers recommended (e.g., MIT, Apache-2.0, CC-BY-4.0) or full SPDX URLs (e.g., http://spdx.org/licenses/MIT). Aligns with Schema.org license property.",
+                "maxLength": 200
+              },
+              "tags": {
+                "type": "array",
+                "description": "Categorization keywords for discovery. Aligns with Schema.org keywords property.",
+                "items": {
+                  "type": "string",
+                  "maxLength": 150
+                },
+                "maxLength": 30
+              }
+            }
+          },
+          "createdAt": {
+            "type": "string",
+            "format": "datetime",
+            "description": "Timestamp when this schema version was created. Immutable once set (ATProto records are permanent)."
+          }
+        }
+      }
+    },
+    "jsonSchemaFormat": {
+      "type": "object",
+      "description": "JSON Schema Draft 7 format for sample type definitions. Used with NDArray shim for array types.",
+      "required": ["$type", "$schema", "type", "properties"],
+      "properties": {
+        "$type": {
+          "type": "string",
+          "const": "ac.foundation.dataset.schema#jsonSchemaFormat"
+        },
+        "$schema": {
+          "type": "string",
+          "const": "http://json-schema.org/draft-07/schema#",
+          "description": "JSON Schema version identifier"
+        },
+        "type": {
+          "type": "string",
+          "const": "object",
+          "description": "Sample types must be objects"
+        },
+        "properties": {
+          "type": "object",
+          "description": "Field definitions for the sample type",
+          "minProperties": 1
+        },
+        "arrayFormatVersions": {
+          "type": "object",
+          "description": "Mapping from array format identifiers to semantic versions. Keys are ac.foundation.dataset.arrayFormat values (e.g., 'ndarrayBytes'), values are semver strings (e.g., '1.0.0'). Foundation.ac maintains canonical shim schemas at https://foundation.ac/schemas/atdata-{format}-bytes/{version}/.",
+          "maxProperties": 10
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.schemaType.json

@@ -0,0 +1,16 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.schemaType",
+  "defs": {
+    "main": {
+      "type": "string",
+      "description": "Schema type identifier for atdata sample definitions. Known values correspond to token definitions in this Lexicon. New schema types can be added as tokens without breaking changes.",
+      "knownValues": ["jsonSchema"],
+      "maxLength": 50
+    },
+    "jsonSchema": {
+      "type": "token",
+      "description": "JSON Schema Draft 7 format for sample type definitions. When schemaType is 'jsonSchema', the schema field must contain an object conforming to ac.foundation.dataset.schema#jsonSchemaFormat."
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.storageBlobs.json

@@ -0,0 +1,24 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.storageBlobs",
+  "defs": {
+    "main": {
+      "type": "object",
+      "description": "Storage via ATProto PDS blobs for WebDataset tar archives. Each blob contains one or more tar files. Used in ac.foundation.dataset.record storage union for maximum decentralization.",
+      "required": [
+        "blobs"
+      ],
+      "properties": {
+        "blobs": {
+          "type": "array",
+          "description": "Array of blob references for WebDataset tar files",
+          "items": {
+            "type": "blob",
+            "description": "Blob reference to a WebDataset tar archive"
+          },
+          "minLength": 1
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.storageExternal.json

@@ -0,0 +1,25 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.storageExternal",
+  "defs": {
+    "main": {
+      "type": "object",
+      "description": "External storage via URLs (S3, HTTP, IPFS, etc.) for WebDataset tar archives. URLs support brace notation for sharding (e.g., 'data-{000000..000099}.tar'). Used in ac.foundation.dataset.record storage union.",
+      "required": [
+        "urls"
+      ],
+      "properties": {
+        "urls": {
+          "type": "array",
+          "description": "WebDataset URLs with optional brace notation for sharded tar files",
+          "items": {
+            "type": "string",
+            "format": "uri",
+            "maxLength": 1000
+          },
+          "minLength": 1
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ndarray_shim.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://foundation.ac/schemas/atdata-ndarray-bytes/1.0.0",
+  "title": "ATDataNDArrayBytes",
+  "description": "Standard definition for numpy NDArray types in JSON Schema, compatible with atdata WebDataset serialization. This type's contents are interpreted as containing the raw bytes data for a serialized numpy NDArray, and serve as a marker for atdata-based code generation to use standard numpy types, rather than generated dataclasses.",
+  "version": "1.0.0",
+  "$defs": {
+    "ndarray": {
+      "type": "string",
+      "format": "byte",
+      "description": "Numpy array serialized using numpy `.npy` format via `np.save` (includes dtype and shape in binary header). When represented in JSON, this is a base64-encoded string. In msgpack, this is raw bytes.",
+      "contentEncoding": "base64",
+      "contentMediaType": "application/octet-stream"
+    }
+  }
+}

atdata/local/__init__.py

@@ -1,24 +1,22 @@
-"""Local storage backend for atdata datasets.
+"""Backward-compatibility shim for atdata.local.
 
-Key classes:
+.. deprecated::
+    Import from ``atdata.index`` and ``atdata.stores`` instead::
 
-- ``Index``: Unified index with pluggable providers (SQLite default),
-  named repositories, and optional atmosphere backend.
-- ``LocalDatasetEntry``: Index entry with ATProto-compatible CIDs.
-- ``S3DataStore``: S3-compatible shard storage.
+        from atdata.index import Index, LocalDatasetEntry
+        from atdata.stores import S3DataStore, LocalDiskStore
 """
 
-from atdata.local._entry import (
+from atdata.index import (
+    Index,
     LocalDatasetEntry,
     BasicIndexEntry,
-    REDIS_KEY_DATASET_ENTRY,
-    REDIS_KEY_SCHEMA,
-)
-from atdata.local._schema import (
     SchemaNamespace,
     SchemaFieldType,
     SchemaField,
     LocalSchemaRecord,
+    REDIS_KEY_DATASET_ENTRY,
+    REDIS_KEY_SCHEMA,
     _ATDATA_URI_PREFIX,
     _LEGACY_URI_PREFIX,
     _kind_str_for_sample_type,
@@ -29,8 +27,8 @@ from atdata.local._schema import (
     _python_type_to_field_type,
     _build_schema_record,
 )
-from atdata.local._index import Index
-from atdata.local._s3 import (
+from atdata.stores import (
+    LocalDiskStore,
     S3DataStore,
     _s3_env,
     _s3_from_credentials,
@@ -44,6 +42,7 @@ from s3fs import S3FileSystem  # noqa: F401 — re-exported for backward compat
 
 __all__ = [
     # Public API
+    "LocalDiskStore",
     "Index",
     "LocalDatasetEntry",
     "BasicIndexEntry",

atdata/local/_repo_legacy.py

@@ -2,8 +2,8 @@
 
 from atdata import Dataset
 
-from atdata.local._entry import LocalDatasetEntry
-from atdata.local._s3 import _s3_env, _s3_from_credentials, _create_s3_write_callbacks
+from atdata.index._entry import LocalDatasetEntry
+from atdata.stores._s3 import _s3_env, _s3_from_credentials, _create_s3_write_callbacks
 
 from pathlib import Path
 from uuid import uuid4
@@ -97,7 +97,7 @@ class Repo:
 
         #
 
-        from atdata.local._index import Index
+        from atdata.index._index import Index
 
         self.index = Index(redis=redis)