atdata 0.2.0a1__py3-none-any.whl → 0.2.2b1__py3-none-any.whl

atdata/__init__.py

@@ -39,20 +39,53 @@ Main Components:
 # Expose components
 
 from .dataset import (
-    PackableSample,
-    SampleBatch,
-    Dataset,
-    packable,
+    DictSample as DictSample,
+    PackableSample as PackableSample,
+    SampleBatch as SampleBatch,
+    Dataset as Dataset,
+    packable as packable,
 )
 
 from .lens import (
-    Lens,
-    LensNetwork,
-    lens,
+    Lens as Lens,
+    LensNetwork as LensNetwork,
+    lens as lens,
 )
 
-# ATProto integration (lazy import to avoid requiring atproto package)
-from . import atmosphere
+from ._hf_api import (
+    load_dataset as load_dataset,
+    DatasetDict as DatasetDict,
+)
+
+from ._protocols import (
+    Packable as Packable,
+    IndexEntry as IndexEntry,
+    AbstractIndex as AbstractIndex,
+    AbstractDataStore as AbstractDataStore,
+    DataSource as DataSource,
+)
 
+from ._sources import (
+    URLSource as URLSource,
+    S3Source as S3Source,
+    BlobSource as BlobSource,
+)
+
+from ._schema_codec import (
+    schema_to_type as schema_to_type,
+)
+
+from ._cid import (
+    generate_cid as generate_cid,
+    verify_cid as verify_cid,
+)
+
+from .promote import (
+    promote_to_atmosphere as promote_to_atmosphere,
+)
+
+# ATProto integration (lazy import to avoid requiring atproto package)
+from . import atmosphere as atmosphere
 
-#
+# CLI entry point
+from .cli import main as main

atdata/_cid.py

@@ -0,0 +1,150 @@
+"""CID (Content Identifier) utilities for atdata.
+
+This module provides utilities for generating ATProto-compatible CIDs from
+data. CIDs are content-addressable identifiers that can be used to uniquely
+identify schemas, datasets, and other records.
+
+The CIDs generated here use:
+- CIDv1 format
+- dag-cbor codec (0x71)
+- SHA-256 hash (0x12)
+
+This ensures compatibility with ATProto's CID requirements and enables
+seamless promotion from local storage to atmosphere (ATProto network).
+
+Example:
+    ::
+
+        >>> schema = {"name": "ImageSample", "version": "1.0.0", "fields": [...]}
+        >>> cid = generate_cid(schema)
+        >>> print(cid)
+        bafyreihffx5a2e7k6r5zqgp5iwpjqr2gfyheqhzqtlxagvqjqyxzqpzqaa
+"""
+
+import hashlib
+from typing import Any
+
+import libipld
+
+
+# CID constants
+CID_VERSION_1 = 0x01
+CODEC_DAG_CBOR = 0x71
+HASH_SHA256 = 0x12
+SHA256_SIZE = 0x20
+
+
+def generate_cid(data: Any) -> str:
+    """Generate an ATProto-compatible CID from arbitrary data.
+
+    The data is first encoded as DAG-CBOR, then hashed with SHA-256,
+    and finally formatted as a CIDv1 string (base32 multibase).
+
+    Args:
+        data: Any data structure that can be encoded as DAG-CBOR.
+            This includes dicts, lists, strings, numbers, bytes, etc.
+
+    Returns:
+        CIDv1 string in base32 multibase format (starts with 'bafy').
+
+    Raises:
+        ValueError: If the data cannot be encoded as DAG-CBOR.
+
+    Example:
+        ::
+
+            >>> generate_cid({"name": "test", "value": 42})
+            'bafyrei...'
+    """
+    # Encode data as DAG-CBOR
+    try:
+        cbor_bytes = libipld.encode_dag_cbor(data)
+    except (TypeError, ValueError, OverflowError) as e:
+        raise ValueError(f"Failed to encode data as DAG-CBOR: {e}") from e
+
+    # Hash with SHA-256
+    sha256_hash = hashlib.sha256(cbor_bytes).digest()
+
+    # Build raw CID bytes:
+    # CIDv1 = version(1) + codec(dag-cbor) + multihash
+    # Multihash = code(sha256) + size(32) + digest
+    raw_cid_bytes = bytes([CID_VERSION_1, CODEC_DAG_CBOR, HASH_SHA256, SHA256_SIZE]) + sha256_hash
+
+    # Encode to base32 multibase string
+    return libipld.encode_cid(raw_cid_bytes)
+
+
+def generate_cid_from_bytes(data_bytes: bytes) -> str:
+    """Generate a CID from raw bytes (already encoded data).
+
+    Use this when you have pre-encoded data (e.g., DAG-CBOR bytes from
+    another source) and want to generate its CID without re-encoding.
+
+    Args:
+        data_bytes: Raw bytes to hash (treated as opaque blob).
+
+    Returns:
+        CIDv1 string in base32 multibase format.
+
+    Example:
+        ::
+
+            >>> cbor_bytes = libipld.encode_dag_cbor({"key": "value"})
+            >>> cid = generate_cid_from_bytes(cbor_bytes)
+    """
+    sha256_hash = hashlib.sha256(data_bytes).digest()
+    raw_cid_bytes = bytes([CID_VERSION_1, CODEC_DAG_CBOR, HASH_SHA256, SHA256_SIZE]) + sha256_hash
+    return libipld.encode_cid(raw_cid_bytes)
+
+
+def verify_cid(cid: str, data: Any) -> bool:
+    """Verify that a CID matches the given data.
+
+    Args:
+        cid: CID string to verify.
+        data: Data that should correspond to the CID.
+
+    Returns:
+        True if the CID matches the data, False otherwise.
+
+    Example:
+        ::
+
+            >>> cid = generate_cid({"name": "test"})
+            >>> verify_cid(cid, {"name": "test"})
+            True
+            >>> verify_cid(cid, {"name": "different"})
+            False
+    """
+    expected_cid = generate_cid(data)
+    return cid == expected_cid
+
+
+def parse_cid(cid: str) -> dict:
+    """Parse a CID string into its components.
+
+    Args:
+        cid: CID string to parse.
+
+    Returns:
+        Dictionary with 'version', 'codec', and 'hash' keys.
+        The 'hash' value is itself a dict with 'code', 'size', and 'digest'.
+
+    Example:
+        ::
+
+            >>> info = parse_cid('bafyrei...')
+            >>> info['version']
+            1
+            >>> info['codec']
+            113  # 0x71 = dag-cbor
+    """
+    return libipld.decode_cid(cid)
+
+
+__all__ = [
+    "generate_cid",
+    "generate_cid_from_bytes",
+    "verify_cid",
+    "parse_cid",
+]