atdata 0.3.0b1__py3-none-any.whl → 0.3.2b1__py3-none-any.whl

atdata/lexicons/ac.foundation.dataset.storageHttp.json

@@ -0,0 +1,45 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.storageHttp",
+  "defs": {
+    "main": {
+      "type": "object",
+      "description": "HTTP/HTTPS storage for WebDataset tar archives. Each shard is listed individually with a checksum for integrity verification. Consumers build brace-expansion patterns on the fly when needed.",
+      "required": [
+        "shards"
+      ],
+      "properties": {
+        "shards": {
+          "type": "array",
+          "description": "Array of shard entries with URL and integrity checksum",
+          "items": {
+            "type": "ref",
+            "ref": "#shardEntry"
+          },
+          "minLength": 1
+        }
+      }
+    },
+    "shardEntry": {
+      "type": "object",
+      "description": "A single HTTP-accessible shard with integrity checksum",
+      "required": [
+        "url",
+        "checksum"
+      ],
+      "properties": {
+        "url": {
+          "type": "string",
+          "format": "uri",
+          "description": "HTTP/HTTPS URL for this WebDataset tar shard",
+          "maxLength": 2000
+        },
+        "checksum": {
+          "type": "ref",
+          "ref": "ac.foundation.dataset.record#shardChecksum",
+          "description": "Content hash for integrity verification"
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.storageS3.json

@@ -0,0 +1,61 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.storageS3",
+  "defs": {
+    "main": {
+      "type": "object",
+      "description": "S3 or S3-compatible storage for WebDataset tar archives. Supports custom endpoints for MinIO, Cloudflare R2, and other S3-compatible services.",
+      "required": [
+        "bucket",
+        "shards"
+      ],
+      "properties": {
+        "bucket": {
+          "type": "string",
+          "description": "S3 bucket name",
+          "maxLength": 255
+        },
+        "region": {
+          "type": "string",
+          "description": "AWS region (e.g., 'us-east-1'). Optional for S3-compatible services.",
+          "maxLength": 50
+        },
+        "endpoint": {
+          "type": "string",
+          "format": "uri",
+          "description": "Custom S3-compatible endpoint URL (e.g., for MinIO, Cloudflare R2). Omit for standard AWS S3.",
+          "maxLength": 500
+        },
+        "shards": {
+          "type": "array",
+          "description": "Array of shard entries with object key and integrity checksum",
+          "items": {
+            "type": "ref",
+            "ref": "#shardEntry"
+          },
+          "minLength": 1
+        }
+      }
+    },
+    "shardEntry": {
+      "type": "object",
+      "description": "A single S3 object shard with integrity checksum",
+      "required": [
+        "key",
+        "checksum"
+      ],
+      "properties": {
+        "key": {
+          "type": "string",
+          "description": "S3 object key for this WebDataset tar shard",
+          "maxLength": 1024
+        },
+        "checksum": {
+          "type": "ref",
+          "ref": "ac.foundation.dataset.record#shardChecksum",
+          "description": "Content hash for integrity verification"
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ndarray_shim.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://foundation.ac/schemas/atdata-ndarray-bytes/1.0.0",
+  "title": "ATDataNDArrayBytes",
+  "description": "Standard definition for numpy NDArray types in JSON Schema, compatible with atdata WebDataset serialization. This type's contents are interpreted as containing the raw bytes data for a serialized numpy NDArray, and serve as a marker for atdata-based code generation to use standard numpy types, rather than generated dataclasses.",
+  "version": "1.0.0",
+  "$defs": {
+    "ndarray": {
+      "type": "string",
+      "format": "byte",
+      "description": "Numpy array serialized using numpy `.npy` format via `np.save` (includes dtype and shape in binary header). When represented in JSON, this is a base64-encoded string. In msgpack, this is raw bytes.",
+      "contentEncoding": "base64",
+      "contentMediaType": "application/octet-stream"
+    }
+  }
+}

atdata/local/__init__.py

@@ -1,24 +1,22 @@
-"""Local storage backend for atdata datasets.
+"""Backward-compatibility shim for atdata.local.
 
-Key classes:
+.. deprecated::
+    Import from ``atdata.index`` and ``atdata.stores`` instead::
 
-- ``Index``: Unified index with pluggable providers (SQLite default),
-  named repositories, and optional atmosphere backend.
-- ``LocalDatasetEntry``: Index entry with ATProto-compatible CIDs.
-- ``S3DataStore``: S3-compatible shard storage.
+        from atdata.index import Index, LocalDatasetEntry
+        from atdata.stores import S3DataStore, LocalDiskStore
 """
 
-from atdata.local._entry import (
+from atdata.index import (
+    Index,
     LocalDatasetEntry,
     BasicIndexEntry,
-    REDIS_KEY_DATASET_ENTRY,
-    REDIS_KEY_SCHEMA,
-)
-from atdata.local._schema import (
     SchemaNamespace,
     SchemaFieldType,
     SchemaField,
     LocalSchemaRecord,
+    REDIS_KEY_DATASET_ENTRY,
+    REDIS_KEY_SCHEMA,
     _ATDATA_URI_PREFIX,
     _LEGACY_URI_PREFIX,
     _kind_str_for_sample_type,
@@ -29,8 +27,8 @@ from atdata.local._schema import (
     _python_type_to_field_type,
     _build_schema_record,
 )
-from atdata.local._index import Index
-from atdata.local._s3 import (
+from atdata.stores import (
+    LocalDiskStore,
     S3DataStore,
     _s3_env,
     _s3_from_credentials,
@@ -44,6 +42,7 @@ from s3fs import S3FileSystem  # noqa: F401 — re-exported for backward compat
 
 __all__ = [
     # Public API
+    "LocalDiskStore",
     "Index",
     "LocalDatasetEntry",
     "BasicIndexEntry",

atdata/local/_repo_legacy.py

@@ -2,8 +2,8 @@
 
 from atdata import Dataset
 
-from atdata.local._entry import LocalDatasetEntry
-from atdata.local._s3 import _s3_env, _s3_from_credentials, _create_s3_write_callbacks
+from atdata.index._entry import LocalDatasetEntry
+from atdata.stores._s3 import _s3_env, _s3_from_credentials, _create_s3_write_callbacks
 
 from pathlib import Path
 from uuid import uuid4
@@ -97,7 +97,7 @@ class Repo:
 
         #
 
-        from atdata.local._index import Index
+        from atdata.index._index import Index
 
         self.index = Index(redis=redis)
 

atdata/manifest/__init__.py

@@ -26,3 +26,7 @@ from ._manifest import MANIFEST_FORMAT_VERSION as MANIFEST_FORMAT_VERSION
 from ._writer import ManifestWriter as ManifestWriter
 from ._query import QueryExecutor as QueryExecutor
 from ._query import SampleLocation as SampleLocation
+from ._proxy import FieldProxy as FieldProxy
+from ._proxy import Predicate as Predicate
+from ._proxy import query_fields as query_fields
+from ._proxy import F as F

atdata/manifest/_proxy.py

@@ -0,0 +1,321 @@
+"""Typed proxy DSL for manifest queries.
+
+Provides ``FieldProxy`` and ``Predicate`` classes that build pandas
+filter expressions with IDE autocomplete and type safety.
+
+Components:
+
+- ``FieldProxy``: Wraps a field name; comparison operators return ``Predicate``
+- ``Predicate``: Composable boolean expression tree; compiles to pandas ops
+- ``query_fields()``: Factory that creates a typed proxy from a sample type
+- ``F``: Untyped convenience proxy (Django-style F expressions)
+
+Examples:
+    >>> Q = query_fields(MySample)
+    >>> pred = (Q.confidence > 0.9) & (Q.label == "dog")
+
+    >>> from atdata.manifest import F
+    >>> pred = (F.confidence > 0.9)
+"""
+
+from __future__ import annotations
+
+import functools
+from typing import Any, Callable, Sequence, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+from ._fields import resolve_manifest_fields
+
+
+class Predicate:
+    """A composable boolean predicate over manifest fields.
+
+    Constructed by comparison operators on ``FieldProxy`` objects.
+    Supports ``&`` (AND), ``|`` (OR), and ``~`` (NOT).
+
+    Call the predicate directly on a DataFrame to evaluate it,
+    or pass it as the ``where`` argument to ``QueryExecutor.query()``.
+
+    Examples:
+        >>> from atdata.manifest import F
+        >>> pred = (F.confidence > 0.9) & (F.label == "dog")
+        >>> pred = (F.score >= 0.5) | (F.label.isin(["cat", "dog"]))
+    """
+
+    __slots__ = ("_kind", "_field", "_op", "_value", "_children", "_child", "_compiled")
+    __hash__ = None  # type: ignore[assignment]
+
+    def __init__(
+        self,
+        kind: str,
+        *,
+        field: str | None = None,
+        op: str | None = None,
+        value: Any = None,
+        children: list[Predicate] | None = None,
+        child: Predicate | None = None,
+    ) -> None:
+        self._kind = kind
+        self._field = field
+        self._op = op
+        self._value = value
+        self._children = children
+        self._child = child
+        self._compiled: Callable[[pd.DataFrame], pd.Series] | None = None
+
+    def __and__(self, other: Predicate) -> Predicate:
+        if not isinstance(other, Predicate):
+            return NotImplemented
+        # Flatten nested ANDs for a cleaner tree
+        left = self._children if self._kind == "and" else [self]
+        right = other._children if other._kind == "and" else [other]
+        return Predicate("and", children=[*left, *right])
+
+    def __or__(self, other: Predicate) -> Predicate:
+        if not isinstance(other, Predicate):
+            return NotImplemented
+        left = self._children if self._kind == "or" else [self]
+        right = other._children if other._kind == "or" else [other]
+        return Predicate("or", children=[*left, *right])
+
+    def __invert__(self) -> Predicate:
+        return Predicate("not", child=self)
+
+    def compile(self) -> Callable[[pd.DataFrame], pd.Series]:
+        """Compile this predicate tree into a callable DataFrame filter.
+
+        Returns:
+            A callable that accepts a ``pd.DataFrame`` and returns a
+            boolean ``pd.Series``.
+        """
+        if self._compiled is not None:
+            return self._compiled
+
+        self._compiled = self._build()
+        return self._compiled
+
+    def _build(self) -> Callable[[pd.DataFrame], pd.Series]:
+        """Recursively build the pandas filter closure."""
+
+        if self._kind == "comparison":
+            field = self._field
+            op = self._op
+            value = self._value
+            return _make_comparison(field, op, value)
+
+        if self._kind == "and":
+            compiled_children = [c.compile() for c in self._children]  # type: ignore[union-attr]
+            return _make_and(compiled_children)
+
+        if self._kind == "or":
+            compiled_children = [c.compile() for c in self._children]  # type: ignore[union-attr]
+            return _make_or(compiled_children)
+
+        if self._kind == "not":
+            compiled_child = self._child.compile()  # type: ignore[union-attr]
+            return _make_not(compiled_child)
+
+        raise ValueError(f"Unknown predicate kind: {self._kind!r}")
+
+    def __call__(self, df: pd.DataFrame) -> pd.Series:
+        """Evaluate this predicate against a DataFrame.
+
+        This makes ``Predicate`` directly usable as a ``where`` argument
+        to ``QueryExecutor.query()`` without any adapter code.
+        """
+        return self.compile()(df)
+
+    def __repr__(self) -> str:
+        if self._kind == "comparison":
+            return f"Predicate({self._field!r} {self._op} {self._value!r})"
+        if self._kind == "not":
+            return f"~{self._child!r}"
+        sep = " & " if self._kind == "and" else " | "
+        parts = sep.join(repr(c) for c in self._children)  # type: ignore[union-attr]
+        return f"({parts})"
+
+
+def _make_comparison(
+    field: str | None, op: str | None, value: Any
+) -> Callable[[pd.DataFrame], pd.Series]:
+    """Create a closure for a single comparison operation."""
+    if op == "gt":
+        return lambda df: df[field] > value
+    if op == "lt":
+        return lambda df: df[field] < value
+    if op == "ge":
+        return lambda df: df[field] >= value
+    if op == "le":
+        return lambda df: df[field] <= value
+    if op == "eq":
+        return lambda df: df[field] == value
+    if op == "ne":
+        return lambda df: df[field] != value
+    if op == "isin":
+        return lambda df: df[field].isin(value)
+    raise ValueError(f"Unknown operator: {op!r}")
+
+
+def _make_and(
+    children: list[Callable[[pd.DataFrame], pd.Series]],
+) -> Callable[[pd.DataFrame], pd.Series]:
+    """Create a closure that ANDs multiple child predicates."""
+
+    def _and(df: pd.DataFrame) -> pd.Series:
+        return functools.reduce(lambda a, b: a & b, (c(df) for c in children))
+
+    return _and
+
+
+def _make_or(
+    children: list[Callable[[pd.DataFrame], pd.Series]],
+) -> Callable[[pd.DataFrame], pd.Series]:
+    """Create a closure that ORs multiple child predicates."""
+
+    def _or(df: pd.DataFrame) -> pd.Series:
+        return functools.reduce(lambda a, b: a | b, (c(df) for c in children))
+
+    return _or
+
+
+def _make_not(
+    child: Callable[[pd.DataFrame], pd.Series],
+) -> Callable[[pd.DataFrame], pd.Series]:
+    """Create a closure that negates a child predicate."""
+    return lambda df: ~child(df)
+
+
+class FieldProxy:
+    """Proxy for a single manifest field.
+
+    Comparison operators return ``Predicate`` objects for composable queries.
+
+    Args:
+        name: The manifest field name (column name in the parquet DataFrame).
+
+    Examples:
+        >>> from atdata.manifest import F
+        >>> pred = F.confidence > 0.9
+        >>> pred = F.label.isin(["dog", "cat"])
+    """
+
+    __slots__ = ("_name",)
+
+    def __init__(self, name: str) -> None:
+        self._name = name
+
+    def __gt__(self, value: Any) -> Predicate:
+        return Predicate("comparison", field=self._name, op="gt", value=value)
+
+    def __lt__(self, value: Any) -> Predicate:
+        return Predicate("comparison", field=self._name, op="lt", value=value)
+
+    def __ge__(self, value: Any) -> Predicate:
+        return Predicate("comparison", field=self._name, op="ge", value=value)
+
+    def __le__(self, value: Any) -> Predicate:
+        return Predicate("comparison", field=self._name, op="le", value=value)
+
+    def __eq__(self, value: Any) -> Predicate:  # type: ignore[override]
+        return Predicate("comparison", field=self._name, op="eq", value=value)
+
+    def __ne__(self, value: Any) -> Predicate:  # type: ignore[override]
+        return Predicate("comparison", field=self._name, op="ne", value=value)
+
+    def isin(self, values: Sequence[Any]) -> Predicate:
+        """Check membership in a set of values.
+
+        Args:
+            values: Collection of values to test membership against.
+
+        Returns:
+            A ``Predicate`` that filters for rows where this field's
+            value is in *values*.
+
+        Examples:
+            >>> pred = F.label.isin(["dog", "cat", "bird"])
+        """
+        return Predicate("comparison", field=self._name, op="isin", value=values)
+
+    def between(self, low: Any, high: Any) -> Predicate:
+        """Check that the field value is within a closed range.
+
+        Shorthand for ``(field >= low) & (field <= high)``.
+
+        Args:
+            low: Lower bound (inclusive).
+            high: Upper bound (inclusive).
+
+        Returns:
+            A ``Predicate`` that filters for rows where this field's
+            value is between *low* and *high* inclusive.
+
+        Examples:
+            >>> pred = F.confidence.between(0.5, 0.9)
+        """
+        return (self >= low) & (self <= high)
+
+    def __repr__(self) -> str:
+        return f"FieldProxy({self._name!r})"
+
+
+def query_fields(sample_type: type) -> Any:
+    """Create a typed field proxy for querying a sample type.
+
+    Returns an object whose attributes are ``FieldProxy`` instances for
+    each manifest-eligible field of *sample_type*. Provides IDE
+    autocomplete when the return type is inferred.
+
+    Args:
+        sample_type: A ``@packable`` or ``PackableSample`` subclass.
+
+    Returns:
+        A proxy object with one ``FieldProxy`` attribute per manifest field.
+
+    Raises:
+        TypeError: If *sample_type* is not a dataclass.
+
+    Examples:
+        >>> Q = query_fields(MySample)
+        >>> pred = (Q.confidence > 0.9) & (Q.label == "dog")
+    """
+    fields = resolve_manifest_fields(sample_type)
+    attrs: dict[str, Any] = {}
+    annotations: dict[str, type] = {}
+    for name in fields:
+        attrs[name] = FieldProxy(name)
+        annotations[name] = FieldProxy
+    attrs["__annotations__"] = annotations
+    attrs["__slots__"] = ()
+    attrs["__repr__"] = (
+        lambda self: f"{sample_type.__name__}Fields({', '.join(annotations)})"
+    )
+
+    proxy_cls = type(f"{sample_type.__name__}Fields", (), attrs)
+    return proxy_cls()
+
+
+class _UntypedFieldProxy:
+    """Untyped convenience proxy for quick field access.
+
+    Attribute access returns a ``FieldProxy`` for any name, without
+    requiring a sample type. Useful for ad-hoc queries where IDE
+    autocomplete is not needed.
+
+    Examples:
+        >>> from atdata.manifest import F
+        >>> pred = (F.confidence > 0.9) & (F.label == "dog")
+    """
+
+    def __getattr__(self, name: str) -> FieldProxy:
+        if name.startswith("_"):
+            raise AttributeError(name)
+        return FieldProxy(name)
+
+    def __repr__(self) -> str:
+        return "F"
+
+
+F = _UntypedFieldProxy()

atdata/promote.py

@@ -6,29 +6,28 @@ federation while maintaining schema consistency.
 
 Examples:
     >>> from atdata.local import Index, Repo
-    >>> from atdata.atmosphere import AtmosphereClient, AtmosphereIndex
+    >>> from atdata.atmosphere import Atmosphere
     >>> from atdata.promote import promote_to_atmosphere
     >>>
     >>> # Setup
     >>> local_index = Index()
-    >>> client = AtmosphereClient()
-    >>> client.login("handle.bsky.social", "app-password")
+    >>> atmo = Atmosphere.login("handle.bsky.social", "app-password")
     >>>
     >>> # Promote a dataset
     >>> entry = local_index.get_dataset("my-dataset")
-    >>> at_uri = promote_to_atmosphere(entry, local_index, client)
+    >>> at_uri = promote_to_atmosphere(entry, local_index, atmo)
 """
 
 from typing import TYPE_CHECKING, Type
 
 if TYPE_CHECKING:
     from .local import LocalDatasetEntry, Index
-    from .atmosphere import AtmosphereClient
+    from .atmosphere import Atmosphere
     from ._protocols import AbstractDataStore, Packable
 
 
 def _find_existing_schema(
-    client: "AtmosphereClient",
+    client: "Atmosphere",
     name: str,
     version: str,
 ) -> str | None:
@@ -55,7 +54,7 @@ def _find_existing_schema(
 def _find_or_publish_schema(
     sample_type: "Type[Packable]",
     version: str,
-    client: "AtmosphereClient",
+    client: "Atmosphere",
     description: str | None = None,
 ) -> str:
     """Find existing schema or publish a new one.
@@ -95,7 +94,7 @@ def _find_or_publish_schema(
 def promote_to_atmosphere(
     local_entry: "LocalDatasetEntry",
     local_index: "Index",
-    atmosphere_client: "AtmosphereClient",
+    atmosphere_client: "Atmosphere",
     *,
     data_store: "AbstractDataStore | None" = None,
     name: str | None = None,
@@ -108,10 +107,15 @@ def promote_to_atmosphere(
     This function takes a locally-indexed dataset and publishes it to ATProto,
     making it discoverable on the federated atmosphere network.
 
+    .. deprecated::
+        Prefer ``Index.promote_entry()`` or ``Index.promote_dataset()``
+        which provide the same functionality through the unified Index
+        interface without requiring separate client and index arguments.
+
     Args:
         local_entry: The LocalDatasetEntry to promote.
         local_index: Local index containing the schema for this entry.
-        atmosphere_client: Authenticated AtmosphereClient.
+        atmosphere_client: Authenticated Atmosphere.
         data_store: Optional data store for copying data to new location.
             If None, the existing data_urls are used as-is.
         name: Override name for the atmosphere record. Defaults to local name.
@@ -128,7 +132,7 @@ def promote_to_atmosphere(
 
     Examples:
         >>> entry = local_index.get_dataset("mnist-train")
-        >>> uri = promote_to_atmosphere(entry, local_index, client)
+        >>> uri = promote_to_atmosphere(entry, local_index, atmo)
         >>> print(uri)
         at://did:plc:abc123/ac.foundation.dataset.datasetIndex/...
     """