atdata 0.3.1b1__py3-none-any.whl → 0.3.2b1__py3-none-any.whl

atdata/lexicons/__init__.py

@@ -16,10 +16,14 @@ Lexicons:
         Extensible token for schema format identifiers.
     ac.foundation.dataset.arrayFormat
         Extensible token for array serialization formats.
-    ac.foundation.dataset.storageExternal
-        External URL-based storage (S3, HTTP, IPFS).
+    ac.foundation.dataset.storageHttp
+        HTTP/HTTPS URL-based storage with per-shard checksums.
+    ac.foundation.dataset.storageS3
+        S3/S3-compatible object storage with per-shard checksums.
     ac.foundation.dataset.storageBlobs
         ATProto PDS blob-based storage.
+    ac.foundation.dataset.storageExternal
+        (Deprecated) External URL-based storage.
     ac.foundation.dataset.getLatestSchema
         XRPC query for fetching the latest schema version.
 
@@ -47,8 +51,10 @@ LEXICON_IDS = (
     f"{NAMESPACE}.lens",
     f"{NAMESPACE}.schemaType",
     f"{NAMESPACE}.arrayFormat",
-    f"{NAMESPACE}.storageExternal",
+    f"{NAMESPACE}.storageHttp",
+    f"{NAMESPACE}.storageS3",
     f"{NAMESPACE}.storageBlobs",
+    f"{NAMESPACE}.storageExternal",  # deprecated
     f"{NAMESPACE}.getLatestSchema",
 )
 

atdata/lexicons/ac.foundation.dataset.lens.json

@@ -24,11 +24,13 @@
           },
           "sourceSchema": {
             "type": "string",
+            "format": "at-uri",
             "description": "AT-URI reference to source schema",
             "maxLength": 500
           },
           "targetSchema": {
             "type": "string",
+            "format": "at-uri",
             "description": "AT-URI reference to target schema",
             "maxLength": 500
           },

atdata/lexicons/ac.foundation.dataset.record.json

@@ -30,7 +30,8 @@
             "type": "union",
             "description": "Storage location for dataset files (WebDataset tar archives)",
             "refs": [
-              "ac.foundation.dataset.storageExternal",
+              "ac.foundation.dataset.storageHttp",
+              "ac.foundation.dataset.storageS3",
               "ac.foundation.dataset.storageBlobs"
             ]
           },
@@ -71,6 +72,26 @@
         }
       }
     },
+    "shardChecksum": {
+      "type": "object",
+      "description": "Content hash for shard integrity verification. Algorithm is flexible to allow SHA-256, BLAKE3, or other hash functions.",
+      "required": [
+        "algorithm",
+        "digest"
+      ],
+      "properties": {
+        "algorithm": {
+          "type": "string",
+          "description": "Hash algorithm identifier (e.g., 'sha256', 'blake3')",
+          "maxLength": 20
+        },
+        "digest": {
+          "type": "string",
+          "description": "Hex-encoded hash digest",
+          "maxLength": 128
+        }
+      }
+    },
     "datasetSize": {
       "type": "object",
       "description": "Information about dataset size",

atdata/lexicons/ac.foundation.dataset.storageBlobs.json

@@ -4,21 +4,43 @@
   "defs": {
     "main": {
       "type": "object",
-      "description": "Storage via ATProto PDS blobs for WebDataset tar archives. Each blob contains one or more tar files. Used in ac.foundation.dataset.record storage union for maximum decentralization.",
+      "description": "Storage via ATProto PDS blobs for WebDataset tar archives. Used in ac.foundation.dataset.record storage union for maximum decentralization.",
       "required": [
         "blobs"
       ],
       "properties": {
         "blobs": {
           "type": "array",
-          "description": "Array of blob references for WebDataset tar files",
+          "description": "Array of blob entries for WebDataset tar files",
           "items": {
-            "type": "blob",
-            "description": "Blob reference to a WebDataset tar archive"
+            "type": "ref",
+            "ref": "#blobEntry"
           },
           "minLength": 1
         }
       }
+    },
+    "blobEntry": {
+      "type": "object",
+      "description": "A single PDS blob shard with optional integrity checksum",
+      "required": [
+        "blob"
+      ],
+      "properties": {
+        "blob": {
+          "type": "blob",
+          "accept": [
+            "application/x-tar"
+          ],
+          "maxSize": 52428800,
+          "description": "Blob reference to a WebDataset tar archive"
+        },
+        "checksum": {
+          "type": "ref",
+          "ref": "ac.foundation.dataset.record#shardChecksum",
+          "description": "Content hash for integrity verification (optional since PDS blobs have built-in CID integrity)"
+        }
+      }
     }
   }
 }

atdata/lexicons/ac.foundation.dataset.storageExternal.json

@@ -4,7 +4,7 @@
   "defs": {
     "main": {
       "type": "object",
-      "description": "External storage via URLs (S3, HTTP, IPFS, etc.) for WebDataset tar archives. URLs support brace notation for sharding (e.g., 'data-{000000..000099}.tar'). Used in ac.foundation.dataset.record storage union.",
+      "description": "(Deprecated: use storageHttp or storageS3 instead.) External storage via URLs for WebDataset tar archives. URLs support brace notation for sharding (e.g., 'data-{000000..000099}.tar').",
       "required": [
         "urls"
       ],

atdata/lexicons/ac.foundation.dataset.storageHttp.json

@@ -0,0 +1,45 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.storageHttp",
+  "defs": {
+    "main": {
+      "type": "object",
+      "description": "HTTP/HTTPS storage for WebDataset tar archives. Each shard is listed individually with a checksum for integrity verification. Consumers build brace-expansion patterns on the fly when needed.",
+      "required": [
+        "shards"
+      ],
+      "properties": {
+        "shards": {
+          "type": "array",
+          "description": "Array of shard entries with URL and integrity checksum",
+          "items": {
+            "type": "ref",
+            "ref": "#shardEntry"
+          },
+          "minLength": 1
+        }
+      }
+    },
+    "shardEntry": {
+      "type": "object",
+      "description": "A single HTTP-accessible shard with integrity checksum",
+      "required": [
+        "url",
+        "checksum"
+      ],
+      "properties": {
+        "url": {
+          "type": "string",
+          "format": "uri",
+          "description": "HTTP/HTTPS URL for this WebDataset tar shard",
+          "maxLength": 2000
+        },
+        "checksum": {
+          "type": "ref",
+          "ref": "ac.foundation.dataset.record#shardChecksum",
+          "description": "Content hash for integrity verification"
+        }
+      }
+    }
+  }
+}

atdata/lexicons/ac.foundation.dataset.storageS3.json

@@ -0,0 +1,61 @@
+{
+  "lexicon": 1,
+  "id": "ac.foundation.dataset.storageS3",
+  "defs": {
+    "main": {
+      "type": "object",
+      "description": "S3 or S3-compatible storage for WebDataset tar archives. Supports custom endpoints for MinIO, Cloudflare R2, and other S3-compatible services.",
+      "required": [
+        "bucket",
+        "shards"
+      ],
+      "properties": {
+        "bucket": {
+          "type": "string",
+          "description": "S3 bucket name",
+          "maxLength": 255
+        },
+        "region": {
+          "type": "string",
+          "description": "AWS region (e.g., 'us-east-1'). Optional for S3-compatible services.",
+          "maxLength": 50
+        },
+        "endpoint": {
+          "type": "string",
+          "format": "uri",
+          "description": "Custom S3-compatible endpoint URL (e.g., for MinIO, Cloudflare R2). Omit for standard AWS S3.",
+          "maxLength": 500
+        },
+        "shards": {
+          "type": "array",
+          "description": "Array of shard entries with object key and integrity checksum",
+          "items": {
+            "type": "ref",
+            "ref": "#shardEntry"
+          },
+          "minLength": 1
+        }
+      }
+    },
+    "shardEntry": {
+      "type": "object",
+      "description": "A single S3 object shard with integrity checksum",
+      "required": [
+        "key",
+        "checksum"
+      ],
+      "properties": {
+        "key": {
+          "type": "string",
+          "description": "S3 object key for this WebDataset tar shard",
+          "maxLength": 1024
+        },
+        "checksum": {
+          "type": "ref",
+          "ref": "ac.foundation.dataset.record#shardChecksum",
+          "description": "Content hash for integrity verification"
+        }
+      }
+    }
+  }
+}

atdata/manifest/__init__.py

@@ -26,3 +26,7 @@ from ._manifest import MANIFEST_FORMAT_VERSION as MANIFEST_FORMAT_VERSION
 from ._writer import ManifestWriter as ManifestWriter
 from ._query import QueryExecutor as QueryExecutor
 from ._query import SampleLocation as SampleLocation
+from ._proxy import FieldProxy as FieldProxy
+from ._proxy import Predicate as Predicate
+from ._proxy import query_fields as query_fields
+from ._proxy import F as F

atdata/manifest/_proxy.py

@@ -0,0 +1,321 @@
+"""Typed proxy DSL for manifest queries.
+
+Provides ``FieldProxy`` and ``Predicate`` classes that build pandas
+filter expressions with IDE autocomplete and type safety.
+
+Components:
+
+- ``FieldProxy``: Wraps a field name; comparison operators return ``Predicate``
+- ``Predicate``: Composable boolean expression tree; compiles to pandas ops
+- ``query_fields()``: Factory that creates a typed proxy from a sample type
+- ``F``: Untyped convenience proxy (Django-style F expressions)
+
+Examples:
+    >>> Q = query_fields(MySample)
+    >>> pred = (Q.confidence > 0.9) & (Q.label == "dog")
+
+    >>> from atdata.manifest import F
+    >>> pred = (F.confidence > 0.9)
+"""
+
+from __future__ import annotations
+
+import functools
+from typing import Any, Callable, Sequence, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+from ._fields import resolve_manifest_fields
+
+
+class Predicate:
+    """A composable boolean predicate over manifest fields.
+
+    Constructed by comparison operators on ``FieldProxy`` objects.
+    Supports ``&`` (AND), ``|`` (OR), and ``~`` (NOT).
+
+    Call the predicate directly on a DataFrame to evaluate it,
+    or pass it as the ``where`` argument to ``QueryExecutor.query()``.
+
+    Examples:
+        >>> from atdata.manifest import F
+        >>> pred = (F.confidence > 0.9) & (F.label == "dog")
+        >>> pred = (F.score >= 0.5) | (F.label.isin(["cat", "dog"]))
+    """
+
+    __slots__ = ("_kind", "_field", "_op", "_value", "_children", "_child", "_compiled")
+    __hash__ = None  # type: ignore[assignment]
+
+    def __init__(
+        self,
+        kind: str,
+        *,
+        field: str | None = None,
+        op: str | None = None,
+        value: Any = None,
+        children: list[Predicate] | None = None,
+        child: Predicate | None = None,
+    ) -> None:
+        self._kind = kind
+        self._field = field
+        self._op = op
+        self._value = value
+        self._children = children
+        self._child = child
+        self._compiled: Callable[[pd.DataFrame], pd.Series] | None = None
+
+    def __and__(self, other: Predicate) -> Predicate:
+        if not isinstance(other, Predicate):
+            return NotImplemented
+        # Flatten nested ANDs for a cleaner tree
+        left = self._children if self._kind == "and" else [self]
+        right = other._children if other._kind == "and" else [other]
+        return Predicate("and", children=[*left, *right])
+
+    def __or__(self, other: Predicate) -> Predicate:
+        if not isinstance(other, Predicate):
+            return NotImplemented
+        left = self._children if self._kind == "or" else [self]
+        right = other._children if other._kind == "or" else [other]
+        return Predicate("or", children=[*left, *right])
+
+    def __invert__(self) -> Predicate:
+        return Predicate("not", child=self)
+
+    def compile(self) -> Callable[[pd.DataFrame], pd.Series]:
+        """Compile this predicate tree into a callable DataFrame filter.
+
+        Returns:
+            A callable that accepts a ``pd.DataFrame`` and returns a
+            boolean ``pd.Series``.
+        """
+        if self._compiled is not None:
+            return self._compiled
+
+        self._compiled = self._build()
+        return self._compiled
+
+    def _build(self) -> Callable[[pd.DataFrame], pd.Series]:
+        """Recursively build the pandas filter closure."""
+
+        if self._kind == "comparison":
+            field = self._field
+            op = self._op
+            value = self._value
+            return _make_comparison(field, op, value)
+
+        if self._kind == "and":
+            compiled_children = [c.compile() for c in self._children]  # type: ignore[union-attr]
+            return _make_and(compiled_children)
+
+        if self._kind == "or":
+            compiled_children = [c.compile() for c in self._children]  # type: ignore[union-attr]
+            return _make_or(compiled_children)
+
+        if self._kind == "not":
+            compiled_child = self._child.compile()  # type: ignore[union-attr]
+            return _make_not(compiled_child)
+
+        raise ValueError(f"Unknown predicate kind: {self._kind!r}")
+
+    def __call__(self, df: pd.DataFrame) -> pd.Series:
+        """Evaluate this predicate against a DataFrame.
+
+        This makes ``Predicate`` directly usable as a ``where`` argument
+        to ``QueryExecutor.query()`` without any adapter code.
+        """
+        return self.compile()(df)
+
+    def __repr__(self) -> str:
+        if self._kind == "comparison":
+            return f"Predicate({self._field!r} {self._op} {self._value!r})"
+        if self._kind == "not":
+            return f"~{self._child!r}"
+        sep = " & " if self._kind == "and" else " | "
+        parts = sep.join(repr(c) for c in self._children)  # type: ignore[union-attr]
+        return f"({parts})"
+
+
+def _make_comparison(
+    field: str | None, op: str | None, value: Any
+) -> Callable[[pd.DataFrame], pd.Series]:
+    """Create a closure for a single comparison operation."""
+    if op == "gt":
+        return lambda df: df[field] > value
+    if op == "lt":
+        return lambda df: df[field] < value
+    if op == "ge":
+        return lambda df: df[field] >= value
+    if op == "le":
+        return lambda df: df[field] <= value
+    if op == "eq":
+        return lambda df: df[field] == value
+    if op == "ne":
+        return lambda df: df[field] != value
+    if op == "isin":
+        return lambda df: df[field].isin(value)
+    raise ValueError(f"Unknown operator: {op!r}")
+
+
+def _make_and(
+    children: list[Callable[[pd.DataFrame], pd.Series]],
+) -> Callable[[pd.DataFrame], pd.Series]:
+    """Create a closure that ANDs multiple child predicates."""
+
+    def _and(df: pd.DataFrame) -> pd.Series:
+        return functools.reduce(lambda a, b: a & b, (c(df) for c in children))
+
+    return _and
+
+
+def _make_or(
+    children: list[Callable[[pd.DataFrame], pd.Series]],
+) -> Callable[[pd.DataFrame], pd.Series]:
+    """Create a closure that ORs multiple child predicates."""
+
+    def _or(df: pd.DataFrame) -> pd.Series:
+        return functools.reduce(lambda a, b: a | b, (c(df) for c in children))
+
+    return _or
+
+
+def _make_not(
+    child: Callable[[pd.DataFrame], pd.Series],
+) -> Callable[[pd.DataFrame], pd.Series]:
+    """Create a closure that negates a child predicate."""
+    return lambda df: ~child(df)
+
+
+class FieldProxy:
+    """Proxy for a single manifest field.
+
+    Comparison operators return ``Predicate`` objects for composable queries.
+
+    Args:
+        name: The manifest field name (column name in the parquet DataFrame).
+
+    Examples:
+        >>> from atdata.manifest import F
+        >>> pred = F.confidence > 0.9
+        >>> pred = F.label.isin(["dog", "cat"])
+    """
+
+    __slots__ = ("_name",)
+
+    def __init__(self, name: str) -> None:
+        self._name = name
+
+    def __gt__(self, value: Any) -> Predicate:
+        return Predicate("comparison", field=self._name, op="gt", value=value)
+
+    def __lt__(self, value: Any) -> Predicate:
+        return Predicate("comparison", field=self._name, op="lt", value=value)
+
+    def __ge__(self, value: Any) -> Predicate:
+        return Predicate("comparison", field=self._name, op="ge", value=value)
+
+    def __le__(self, value: Any) -> Predicate:
+        return Predicate("comparison", field=self._name, op="le", value=value)
+
+    def __eq__(self, value: Any) -> Predicate:  # type: ignore[override]
+        return Predicate("comparison", field=self._name, op="eq", value=value)
+
+    def __ne__(self, value: Any) -> Predicate:  # type: ignore[override]
+        return Predicate("comparison", field=self._name, op="ne", value=value)
+
+    def isin(self, values: Sequence[Any]) -> Predicate:
+        """Check membership in a set of values.
+
+        Args:
+            values: Collection of values to test membership against.
+
+        Returns:
+            A ``Predicate`` that filters for rows where this field's
+            value is in *values*.
+
+        Examples:
+            >>> pred = F.label.isin(["dog", "cat", "bird"])
+        """
+        return Predicate("comparison", field=self._name, op="isin", value=values)
+
+    def between(self, low: Any, high: Any) -> Predicate:
+        """Check that the field value is within a closed range.
+
+        Shorthand for ``(field >= low) & (field <= high)``.
+
+        Args:
+            low: Lower bound (inclusive).
+            high: Upper bound (inclusive).
+
+        Returns:
+            A ``Predicate`` that filters for rows where this field's
+            value is between *low* and *high* inclusive.
+
+        Examples:
+            >>> pred = F.confidence.between(0.5, 0.9)
+        """
+        return (self >= low) & (self <= high)
+
+    def __repr__(self) -> str:
+        return f"FieldProxy({self._name!r})"
+
+
+def query_fields(sample_type: type) -> Any:
+    """Create a typed field proxy for querying a sample type.
+
+    Returns an object whose attributes are ``FieldProxy`` instances for
+    each manifest-eligible field of *sample_type*. Provides IDE
+    autocomplete when the return type is inferred.
+
+    Args:
+        sample_type: A ``@packable`` or ``PackableSample`` subclass.
+
+    Returns:
+        A proxy object with one ``FieldProxy`` attribute per manifest field.
+
+    Raises:
+        TypeError: If *sample_type* is not a dataclass.
+
+    Examples:
+        >>> Q = query_fields(MySample)
+        >>> pred = (Q.confidence > 0.9) & (Q.label == "dog")
+    """
+    fields = resolve_manifest_fields(sample_type)
+    attrs: dict[str, Any] = {}
+    annotations: dict[str, type] = {}
+    for name in fields:
+        attrs[name] = FieldProxy(name)
+        annotations[name] = FieldProxy
+    attrs["__annotations__"] = annotations
+    attrs["__slots__"] = ()
+    attrs["__repr__"] = (
+        lambda self: f"{sample_type.__name__}Fields({', '.join(annotations)})"
+    )
+
+    proxy_cls = type(f"{sample_type.__name__}Fields", (), attrs)
+    return proxy_cls()
+
+
+class _UntypedFieldProxy:
+    """Untyped convenience proxy for quick field access.
+
+    Attribute access returns a ``FieldProxy`` for any name, without
+    requiring a sample type. Useful for ad-hoc queries where IDE
+    autocomplete is not needed.
+
+    Examples:
+        >>> from atdata.manifest import F
+        >>> pred = (F.confidence > 0.9) & (F.label == "dog")
+    """
+
+    def __getattr__(self, name: str) -> FieldProxy:
+        if name.startswith("_"):
+            raise AttributeError(name)
+        return FieldProxy(name)
+
+    def __repr__(self) -> str:
+        return "F"
+
+
+F = _UntypedFieldProxy()

atdata/repository.py

@@ -210,14 +210,26 @@ class _AtmosphereBackend:
         *,
         name: str,
         schema_ref: str | None = None,
+        data_urls: list[str] | None = None,
+        blob_refs: list[dict] | None = None,
         **kwargs: Any,
     ) -> Any:
         """Insert a dataset into ATProto.
 
+        When *blob_refs* is provided the record uses ``storageBlobs`` with
+        embedded blob reference objects so the PDS retains the uploaded blobs.
+
+        When *data_urls* is provided (without *blob_refs*) the record uses
+        ``storageExternal`` with those URLs.
+
         Args:
             ds: The Dataset to publish.
             name: Human-readable name.
             schema_ref: Optional schema AT URI. If None, auto-publishes schema.
+            data_urls: Explicit shard URLs to store in the record.  When
+                provided, these replace whatever ``ds.url`` contains.
+            blob_refs: Pre-uploaded blob reference dicts from
+                ``PDSBlobStore``.  Takes precedence over *data_urls*.
             **kwargs: Additional options (description, tags, license).
 
         Returns:
@@ -226,15 +238,53 @@ class _AtmosphereBackend:
         self._ensure_loaders()
         from .atmosphere import AtmosphereIndexEntry
 
-        uri = self._dataset_publisher.publish(
-            ds,
-            name=name,
-            schema_uri=schema_ref,
-            description=kwargs.get("description"),
-            tags=kwargs.get("tags"),
-            license=kwargs.get("license"),
-            auto_publish_schema=(schema_ref is None),
-        )
+        if blob_refs is not None or data_urls is not None:
+            # Ensure schema is published first
+            if schema_ref is None:
+                from .atmosphere import SchemaPublisher
+
+                sp = SchemaPublisher(self.client)
+                schema_uri_obj = sp.publish(
+                    ds.sample_type,
+                    version=kwargs.get("schema_version", "1.0.0"),
+                )
+                schema_ref = str(schema_uri_obj)
+
+            metadata = kwargs.get("metadata")
+            if metadata is None and hasattr(ds, "_metadata"):
+                metadata = ds._metadata
+
+            if blob_refs is not None:
+                uri = self._dataset_publisher.publish_with_blob_refs(
+                    blob_refs=blob_refs,
+                    schema_uri=schema_ref,
+                    name=name,
+                    description=kwargs.get("description"),
+                    tags=kwargs.get("tags"),
+                    license=kwargs.get("license"),
+                    metadata=metadata,
+                )
+            else:
+                uri = self._dataset_publisher.publish_with_urls(
+                    urls=data_urls,
+                    schema_uri=schema_ref,
+                    name=name,
+                    description=kwargs.get("description"),
+                    tags=kwargs.get("tags"),
+                    license=kwargs.get("license"),
+                    metadata=metadata,
+                )
+        else:
+            uri = self._dataset_publisher.publish(
+                ds,
+                name=name,
+                schema_uri=schema_ref,
+                description=kwargs.get("description"),
+                tags=kwargs.get("tags"),
+                license=kwargs.get("license"),
+                auto_publish_schema=(schema_ref is None),
+            )
+
         record = self._dataset_loader.get(uri)
         return AtmosphereIndexEntry(str(uri), record)