atdata 0.3.0b1__py3-none-any.whl → 0.3.2b1__py3-none-any.whl

atdata/{local → index}/_schema.py

@@ -26,7 +26,7 @@ from typing import (
 T = TypeVar("T", bound=Packable)
 
 # URI scheme prefixes
-_ATDATA_URI_PREFIX = "atdata://local/sampleSchema/"
+_ATDATA_URI_PREFIX = "atdata://local/schema/"
 _LEGACY_URI_PREFIX = "local://schemas/"
 
 
@@ -37,7 +37,7 @@ class SchemaNamespace:
     Supports attribute access, iteration, ``len()``, and ``in`` checks.
 
     Examples:
-        >>> index.load_schema("atdata://local/sampleSchema/MySample@1.0.0")
+        >>> index.load_schema("atdata://local/schema/MySample@1.0.0")
         >>> MyType = index.types.MySample
         >>> sample = MyType(field1="hello", field2=42)
 
@@ -207,7 +207,7 @@ class LocalSchemaRecord:
     """List of field definitions."""
 
     ref: str
-    """Schema reference URI (atdata://local/sampleSchema/{name}@{version})."""
+    """Schema reference URI (atdata://local/schema/{name}@{version})."""
 
     description: Optional[str] = None
     """Human-readable description."""
@@ -259,7 +259,7 @@ def _kind_str_for_sample_type(st: Type[Packable]) -> str:
 
 
 def _schema_ref_from_type(sample_type: Type[Packable], version: str) -> str:
-    """Generate 'atdata://local/sampleSchema/{name}@{version}' reference."""
+    """Generate 'atdata://local/schema/{name}@{version}' reference."""
     return _make_schema_ref(sample_type.__name__, version)
 
 
@@ -271,7 +271,7 @@ def _make_schema_ref(name: str, version: str) -> str:
 def _parse_schema_ref(ref: str) -> tuple[str, str]:
     """Parse schema reference into (name, version).
 
-    Supports both new format: 'atdata://local/sampleSchema/{name}@{version}'
+    Supports both new format: 'atdata://local/schema/{name}@{version}'
     and legacy format: 'local://schemas/{module.Class}@{version}'
     """
     if ref.startswith(_ATDATA_URI_PREFIX):

atdata/lexicons/__init__.py

@@ -0,0 +1,127 @@
+"""ATProto Lexicon definitions for the atdata federation.
+
+This package contains the canonical Lexicon JSON files for the
+``ac.foundation.dataset`` namespace. These define the ATProto record
+types used by atdata for publishing schemas, datasets, and lenses
+to the AT Protocol network.
+
+Lexicons:
+    ac.foundation.dataset.schema
+        Versioned sample type definitions (PackableSample schemas).
+    ac.foundation.dataset.record
+        Dataset index records pointing to WebDataset storage.
+    ac.foundation.dataset.lens
+        Bidirectional transformations between schemas.
+    ac.foundation.dataset.schemaType
+        Extensible token for schema format identifiers.
+    ac.foundation.dataset.arrayFormat
+        Extensible token for array serialization formats.
+    ac.foundation.dataset.storageHttp
+        HTTP/HTTPS URL-based storage with per-shard checksums.
+    ac.foundation.dataset.storageS3
+        S3/S3-compatible object storage with per-shard checksums.
+    ac.foundation.dataset.storageBlobs
+        ATProto PDS blob-based storage.
+    ac.foundation.dataset.storageExternal
+        (Deprecated) External URL-based storage.
+    ac.foundation.dataset.getLatestSchema
+        XRPC query for fetching the latest schema version.
+
+The ``ndarray_shim.json`` file defines the standard NDArray type
+for use within JSON Schema definitions.
+
+Examples:
+    >>> from atdata.lexicons import load_lexicon
+    >>> schema_lex = load_lexicon("ac.foundation.dataset.schema")
+    >>> schema_lex["id"]
+    'ac.foundation.dataset.schema'
+"""
+
+import json
+from importlib import resources
+from functools import lru_cache
+from typing import Any
+
+
+NAMESPACE = "ac.foundation.dataset"
+
+LEXICON_IDS = (
+    f"{NAMESPACE}.schema",
+    f"{NAMESPACE}.record",
+    f"{NAMESPACE}.lens",
+    f"{NAMESPACE}.schemaType",
+    f"{NAMESPACE}.arrayFormat",
+    f"{NAMESPACE}.storageHttp",
+    f"{NAMESPACE}.storageS3",
+    f"{NAMESPACE}.storageBlobs",
+    f"{NAMESPACE}.storageExternal",  # deprecated
+    f"{NAMESPACE}.getLatestSchema",
+)
+
+
+@lru_cache(maxsize=16)
+def load_lexicon(lexicon_id: str) -> dict[str, Any]:
+    """Load a lexicon definition by its NSID.
+
+    Args:
+        lexicon_id: The lexicon NSID, e.g. ``"ac.foundation.dataset.schema"``.
+
+    Returns:
+        Parsed JSON dictionary containing the lexicon definition.
+
+    Raises:
+        FileNotFoundError: If no lexicon file exists for the given ID.
+
+    Examples:
+        >>> lex = load_lexicon("ac.foundation.dataset.schema")
+        >>> lex["defs"]["main"]["type"]
+        'record'
+    """
+    filename = f"{lexicon_id}.json"
+    ref = resources.files(__package__).joinpath(filename)
+    try:
+        text = ref.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        raise FileNotFoundError(
+            f"No lexicon file found for '{lexicon_id}'. "
+            f"Expected {filename} in {__package__}."
+        ) from None
+    return json.loads(text)
+
+
+@lru_cache(maxsize=1)
+def load_ndarray_shim() -> dict[str, Any]:
+    """Load the NDArray JSON Schema shim definition.
+
+    Returns:
+        Parsed JSON dictionary containing the NDArray shim schema.
+
+    Examples:
+        >>> shim = load_ndarray_shim()
+        >>> shim["$defs"]["ndarray"]["type"]
+        'string'
+    """
+    ref = resources.files(__package__).joinpath("ndarray_shim.json")
+    return json.loads(ref.read_text(encoding="utf-8"))
+
+
+def list_lexicons() -> tuple[str, ...]:
+    """Return the tuple of all known lexicon NSIDs.
+
+    Returns:
+        Tuple of lexicon ID strings.
+
+    Examples:
+        >>> "ac.foundation.dataset.schema" in list_lexicons()
+        True
+    """
+    return LEXICON_IDS
+
+
+__all__ = [
+    "NAMESPACE",
+    "LEXICON_IDS",
+    "load_lexicon",
+    "load_ndarray_shim",
+    "list_lexicons",
+]

atdata/lexicons/ac.foundation.dataset.arrayFormat.json

@@ -0,0 +1,16 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.arrayFormat",
+  "defs": {
+    "main": {
+      "type": "string",
+      "description": "Array serialization format identifier for NDArray fields in sample schemas. Known values correspond to token definitions in this Lexicon. Each format has versioned specifications maintained by foundation.ac at canonical URLs.",
+      "knownValues": ["ndarrayBytes"],
+      "maxLength": 50
+    },
+    "ndarrayBytes": {
+      "type": "token",
+      "description": "Numpy .npy binary format for NDArray serialization. Stores arrays with dtype and shape in binary header. Versions maintained at https://foundation.ac/schemas/atdata-ndarray-bytes/{version}/"
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.getLatestSchema.json

@@ -0,0 +1,78 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.getLatestSchema",
+  "defs": {
+    "main": {
+      "type": "query",
+      "description": "Get the latest version of a sample schema by its permanent NSID identifier",
+      "parameters": {
+        "type": "params",
+        "required": [
+          "schemaId"
+        ],
+        "properties": {
+          "schemaId": {
+            "type": "string",
+            "description": "The permanent NSID identifier for the schema (the {NSID} part of the rkey {NSID}@{semver})",
+            "maxLength": 500
+          }
+        }
+      },
+      "output": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "required": [
+            "uri",
+            "version",
+            "record"
+          ],
+          "properties": {
+            "uri": {
+              "type": "string",
+              "description": "AT-URI of the latest schema version",
+              "maxLength": 500
+            },
+            "version": {
+              "type": "string",
+              "description": "Semantic version of the latest schema",
+              "maxLength": 20
+            },
+            "record": {
+              "type": "ref",
+              "ref": "ac.foundation.dataset.schema",
+              "description": "The full schema record"
+            },
+            "allVersions": {
+              "type": "array",
+              "description": "All available versions (optional, sorted by semver descending)",
+              "items": {
+                "type": "object",
+                "required": [
+                  "uri",
+                  "version"
+                ],
+                "properties": {
+                  "uri": {
+                    "type": "string",
+                    "maxLength": 500
+                  },
+                  "version": {
+                    "type": "string",
+                    "maxLength": 20
+                  }
+                }
+              }
+            }
+          }
+        }
+      },
+      "errors": [
+        {
+          "name": "SchemaNotFound",
+          "description": "No schema found with the given NSID"
+        }
+      ]
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.lens.json

@@ -0,0 +1,101 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.lens",
+  "defs": {
+    "main": {
+      "type": "record",
+      "description": "Bidirectional transformation (Lens) between two sample types, with code stored in external repositories",
+      "key": "tid",
+      "record": {
+        "type": "object",
+        "required": [
+          "name",
+          "sourceSchema",
+          "targetSchema",
+          "getterCode",
+          "putterCode",
+          "createdAt"
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "Human-readable lens name",
+            "maxLength": 100
+          },
+          "sourceSchema": {
+            "type": "string",
+            "format": "at-uri",
+            "description": "AT-URI reference to source schema",
+            "maxLength": 500
+          },
+          "targetSchema": {
+            "type": "string",
+            "format": "at-uri",
+            "description": "AT-URI reference to target schema",
+            "maxLength": 500
+          },
+          "description": {
+            "type": "string",
+            "description": "What this transformation does",
+            "maxLength": 1000
+          },
+          "getterCode": {
+            "type": "ref",
+            "ref": "#codeReference",
+            "description": "Code reference for getter function (Source -> Target)"
+          },
+          "putterCode": {
+            "type": "ref",
+            "ref": "#codeReference",
+            "description": "Code reference for putter function (Target, Source -> Source)"
+          },
+          "language": {
+            "type": "string",
+            "description": "Programming language of the lens implementation (e.g., 'python', 'typescript')",
+            "maxLength": 50
+          },
+          "metadata": {
+            "type": "object",
+            "description": "Arbitrary metadata (author, performance notes, etc.)"
+          },
+          "createdAt": {
+            "type": "string",
+            "format": "datetime",
+            "description": "Timestamp when this lens was created"
+          }
+        }
+      }
+    },
+    "codeReference": {
+      "type": "object",
+      "description": "Reference to code in an external repository (GitHub, tangled.org, etc.)",
+      "required": [
+        "repository",
+        "commit",
+        "path"
+      ],
+      "properties": {
+        "repository": {
+          "type": "string",
+          "description": "Repository URL (e.g., 'https://github.com/user/repo' or 'at://did/tangled.repo/...')",
+          "maxLength": 500
+        },
+        "commit": {
+          "type": "string",
+          "description": "Git commit hash (ensures immutability)",
+          "maxLength": 40
+        },
+        "path": {
+          "type": "string",
+          "description": "Path to function within repository (e.g., 'lenses/vision.py:rgb_to_grayscale')",
+          "maxLength": 500
+        },
+        "branch": {
+          "type": "string",
+          "description": "Optional branch name (for reference, commit hash is authoritative)",
+          "maxLength": 100
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.record.json

@@ -0,0 +1,117 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.record",
+  "defs": {
+    "main": {
+      "type": "record",
+      "description": "Index record for a WebDataset-backed dataset with references to storage location and sample schema",
+      "key": "tid",
+      "record": {
+        "type": "object",
+        "required": [
+          "name",
+          "schemaRef",
+          "storage",
+          "createdAt"
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "Human-readable dataset name",
+            "maxLength": 200
+          },
+          "schemaRef": {
+            "type": "string",
+            "format": "at-uri",
+            "description": "AT-URI reference to the schema record for this dataset's samples",
+            "maxLength": 500
+          },
+          "storage": {
+            "type": "union",
+            "description": "Storage location for dataset files (WebDataset tar archives)",
+            "refs": [
+              "ac.foundation.dataset.storageHttp",
+              "ac.foundation.dataset.storageS3",
+              "ac.foundation.dataset.storageBlobs"
+            ]
+          },
+          "description": {
+            "type": "string",
+            "description": "Human-readable description of the dataset",
+            "maxLength": 5000
+          },
+          "metadata": {
+            "type": "bytes",
+            "description": "Msgpack-encoded metadata dict for arbitrary extended key-value pairs. Use this for additional metadata beyond the core top-level fields (license, tags, size). Top-level fields are preferred for discoverable/searchable metadata.",
+            "maxLength": 100000
+          },
+          "tags": {
+            "type": "array",
+            "description": "Searchable tags for dataset discovery. Aligns with Schema.org keywords property.",
+            "items": {
+              "type": "string",
+              "maxLength": 150
+            },
+            "maxLength": 30
+          },
+          "size": {
+            "type": "ref",
+            "ref": "#datasetSize",
+            "description": "Dataset size information (optional)"
+          },
+          "license": {
+            "type": "string",
+            "description": "License identifier or URL. SPDX identifiers recommended (e.g., MIT, Apache-2.0, CC-BY-4.0) or full SPDX URLs (e.g., http://spdx.org/licenses/MIT). Aligns with Schema.org license property.",
+            "maxLength": 200
+          },
+          "createdAt": {
+            "type": "string",
+            "format": "datetime",
+            "description": "Timestamp when this dataset record was created"
+          }
+        }
+      }
+    },
+    "shardChecksum": {
+      "type": "object",
+      "description": "Content hash for shard integrity verification. Algorithm is flexible to allow SHA-256, BLAKE3, or other hash functions.",
+      "required": [
+        "algorithm",
+        "digest"
+      ],
+      "properties": {
+        "algorithm": {
+          "type": "string",
+          "description": "Hash algorithm identifier (e.g., 'sha256', 'blake3')",
+          "maxLength": 20
+        },
+        "digest": {
+          "type": "string",
+          "description": "Hex-encoded hash digest",
+          "maxLength": 128
+        }
+      }
+    },
+    "datasetSize": {
+      "type": "object",
+      "description": "Information about dataset size",
+      "properties": {
+        "samples": {
+          "type": "integer",
+          "description": "Total number of samples in the dataset",
+          "minimum": 0
+        },
+        "bytes": {
+          "type": "integer",
+          "description": "Total size in bytes",
+          "minimum": 0
+        },
+        "shards": {
+          "type": "integer",
+          "description": "Number of WebDataset shards",
+          "minimum": 1
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.schema.json

@@ -0,0 +1,107 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.schema",
+  "defs": {
+    "main": {
+      "type": "record",
+      "description": "Definition of a PackableSample-compatible sample type. Supports versioning via rkey format: {NSID}@{semver}. Schema format is extensible via union type.",
+      "key": "any",
+      "record": {
+        "type": "object",
+        "required": [
+          "name",
+          "version",
+          "schemaType",
+          "schema",
+          "createdAt"
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "Human-readable display name for this sample type. Used for documentation and UI. The NSID in the record URI provides unique identification; name collisions across NSIDs are acceptable.",
+            "maxLength": 100
+          },
+          "version": {
+            "type": "string",
+            "description": "Semantic version (e.g., '1.0.0')",
+            "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$",
+            "maxLength": 100
+          },
+          "schemaType": {
+            "type": "ref",
+            "ref": "ac.foundation.dataset.schemaType",
+            "description": "Type of schema definition. This field indicates which union member is present in the schema field."
+          },
+          "schema": {
+            "type": "union",
+            "refs": ["ac.foundation.dataset.schema#jsonSchemaFormat"],
+            "closed": false,
+            "description": "Schema definition for this sample type. Currently supports JSON Schema Draft 7. Union allows for future schema formats (Avro, Protobuf, etc.) without breaking changes."
+          },
+          "description": {
+            "type": "string",
+            "description": "Human-readable description of what this sample type represents",
+            "maxLength": 5000
+          },
+          "metadata": {
+            "type": "object",
+            "description": "Optional metadata about this schema. Common fields include license and tags, but any additional fields are permitted.",
+            "maxProperties": 50,
+            "properties": {
+              "license": {
+                "type": "string",
+                "description": "License identifier or URL. SPDX identifiers recommended (e.g., MIT, Apache-2.0, CC-BY-4.0) or full SPDX URLs (e.g., http://spdx.org/licenses/MIT). Aligns with Schema.org license property.",
+                "maxLength": 200
+              },
+              "tags": {
+                "type": "array",
+                "description": "Categorization keywords for discovery. Aligns with Schema.org keywords property.",
+                "items": {
+                  "type": "string",
+                  "maxLength": 150
+                },
+                "maxLength": 30
+              }
+            }
+          },
+          "createdAt": {
+            "type": "string",
+            "format": "datetime",
+            "description": "Timestamp when this schema version was created. Immutable once set (ATProto records are permanent)."
+          }
+        }
+      }
+    },
+    "jsonSchemaFormat": {
+      "type": "object",
+      "description": "JSON Schema Draft 7 format for sample type definitions. Used with NDArray shim for array types.",
+      "required": ["$type", "$schema", "type", "properties"],
+      "properties": {
+        "$type": {
+          "type": "string",
+          "const": "ac.foundation.dataset.schema#jsonSchemaFormat"
+        },
+        "$schema": {
+          "type": "string",
+          "const": "http://json-schema.org/draft-07/schema#",
+          "description": "JSON Schema version identifier"
+        },
+        "type": {
+          "type": "string",
+          "const": "object",
+          "description": "Sample types must be objects"
+        },
+        "properties": {
+          "type": "object",
+          "description": "Field definitions for the sample type",
+          "minProperties": 1
+        },
+        "arrayFormatVersions": {
+          "type": "object",
+          "description": "Mapping from array format identifiers to semantic versions. Keys are ac.foundation.dataset.arrayFormat values (e.g., 'ndarrayBytes'), values are semver strings (e.g., '1.0.0'). Foundation.ac maintains canonical shim schemas at https://foundation.ac/schemas/atdata-{format}-bytes/{version}/.",
+          "maxProperties": 10
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.schemaType.json

@@ -0,0 +1,16 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.schemaType",
+  "defs": {
+    "main": {
+      "type": "string",
+      "description": "Schema type identifier for atdata sample definitions. Known values correspond to token definitions in this Lexicon. New schema types can be added as tokens without breaking changes.",
+      "knownValues": ["jsonSchema"],
+      "maxLength": 50
+    },
+    "jsonSchema": {
+      "type": "token",
+      "description": "JSON Schema Draft 7 format for sample type definitions. When schemaType is 'jsonSchema', the schema field must contain an object conforming to ac.foundation.dataset.schema#jsonSchemaFormat."
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.storageBlobs.json

@@ -0,0 +1,46 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.storageBlobs",
+  "defs": {
+    "main": {
+      "type": "object",
+      "description": "Storage via ATProto PDS blobs for WebDataset tar archives. Used in ac.foundation.dataset.record storage union for maximum decentralization.",
+      "required": [
+        "blobs"
+      ],
+      "properties": {
+        "blobs": {
+          "type": "array",
+          "description": "Array of blob entries for WebDataset tar files",
+          "items": {
+            "type": "ref",
+            "ref": "#blobEntry"
+          },
+          "minLength": 1
+        }
+      }
+    },
+    "blobEntry": {
+      "type": "object",
+      "description": "A single PDS blob shard with optional integrity checksum",
+      "required": [
+        "blob"
+      ],
+      "properties": {
+        "blob": {
+          "type": "blob",
+          "accept": [
+            "application/x-tar"
+          ],
+          "maxSize": 52428800,
+          "description": "Blob reference to a WebDataset tar archive"
+        },
+        "checksum": {
+          "type": "ref",
+          "ref": "ac.foundation.dataset.record#shardChecksum",
+          "description": "Content hash for integrity verification (optional since PDS blobs have built-in CID integrity)"
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.storageExternal.json

@@ -0,0 +1,25 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.storageExternal",
+  "defs": {
+    "main": {
+      "type": "object",
+      "description": "(Deprecated: use storageHttp or storageS3 instead.) External storage via URLs for WebDataset tar archives. URLs support brace notation for sharding (e.g., 'data-{000000..000099}.tar').",
+      "required": [
+        "urls"
+      ],
+      "properties": {
+        "urls": {
+          "type": "array",
+          "description": "WebDataset URLs with optional brace notation for sharded tar files",
+          "items": {
+            "type": "string",
+            "format": "uri",
+            "maxLength": 1000
+          },
+          "minLength": 1
+        }
+      }
+    }
+  }
+}