chuk-artifacts 0.1.0__py3-none-any.whl

chuk_artifacts/providers/filesystem.py

@@ -0,0 +1,453 @@
+# ===========================================================================
+# chuk_artifacts/providers/filesystem.py
+# ===========================================================================
+"""Local-filesystem artefact store.
+
+Objects are written relative to $ARTIFACT_FS_ROOT (default ./artifacts).
+Presigned URLs use the *file://* scheme so callers can still download.
+Includes comprehensive S3-compatible methods and proper error handling.
+"""
+from __future__ import annotations
+
+import os, json, asyncio, time, uuid, stat, hashlib
+from pathlib import Path
+from contextlib import asynccontextmanager
+from typing import Any, Dict, Callable, AsyncContextManager, List, Optional
+from datetime import datetime
+
+_ROOT = Path(os.getenv("ARTIFACT_FS_ROOT", "./artifacts")).expanduser()
+
+
+class _FilesystemClient:
+    """Mimics the S3 surface ArtifactStore depends on with filesystem backend."""
+
+    def __init__(self, root: Path = _ROOT):
+        self._root = Path(root).expanduser().resolve()
+        self._closed = False
+        self._lock = asyncio.Lock()
+        
+        # Ensure root directory exists
+        self._root.mkdir(parents=True, exist_ok=True)
+
+    def _get_object_path(self, bucket: str, key: str) -> Path:
+        """Get the filesystem path for an object, incorporating bucket as subdirectory."""
+        # Use bucket as a subdirectory to maintain some S3-like organization
+        return self._root / bucket / key
+
+    def _get_metadata_path(self, object_path: Path) -> Path:
+        """Get the metadata file path for an object."""
+        return object_path.with_suffix(object_path.suffix + ".meta.json")
+
+    async def _ensure_parent_dir(self, path: Path):
+        """Ensure parent directory exists."""
+        await asyncio.to_thread(path.parent.mkdir, parents=True, exist_ok=True)
+
+    async def _write_metadata(self, meta_path: Path, content_type: str, metadata: Dict[str, str], size: int, etag: str):
+        """Write metadata file."""
+        meta_data = {
+            "content_type": content_type,
+            "metadata": metadata,
+            "size": size,
+            "etag": etag,
+            "last_modified": datetime.utcnow().isoformat() + "Z",
+            "created_at": datetime.utcnow().isoformat() + "Z"
+        }
+        meta_json = json.dumps(meta_data, indent=2)
+        await asyncio.to_thread(meta_path.write_text, meta_json, encoding='utf-8')
+
+    async def _read_metadata(self, meta_path: Path) -> Dict[str, Any]:
+        """Read metadata file."""
+        try:
+            content = await asyncio.to_thread(meta_path.read_text, encoding='utf-8')
+            return json.loads(content)
+        except (FileNotFoundError, json.JSONDecodeError):
+            return {}
+
+    # ------------------------------------------------------------
+    # Core S3-compatible methods
+    # ------------------------------------------------------------
+
+    async def put_object(
+        self, 
+        *, 
+        Bucket: str,           # noqa: N803
+        Key: str,              # noqa: N803  
+        Body: bytes,           # noqa: N803
+        ContentType: str, 
+        Metadata: Dict[str, str]  # noqa: N803
+    ):
+        """Store object in filesystem with metadata."""
+        if self._closed:
+            raise RuntimeError("Client has been closed")
+
+        object_path = self._get_object_path(Bucket, Key)
+        meta_path = self._get_metadata_path(object_path)
+        
+        # Generate ETag (MD5 hash like S3)
+        etag = hashlib.md5(Body).hexdigest()
+        
+        async with self._lock:
+            await self._ensure_parent_dir(object_path)
+            await asyncio.to_thread(object_path.write_bytes, Body)
+            await self._write_metadata(meta_path, ContentType, Metadata, len(Body), etag)
+
+        return {
+            "ResponseMetadata": {"HTTPStatusCode": 200},
+            "ETag": f'"{etag}"'
+        }
+
+    async def get_object(
+        self, 
+        *, 
+        Bucket: str,  # noqa: N803
+        Key: str      # noqa: N803
+    ):
+        """Retrieve object from filesystem."""
+        if self._closed:
+            raise RuntimeError("Client has been closed")
+
+        object_path = self._get_object_path(Bucket, Key)
+        meta_path = self._get_metadata_path(object_path)
+
+        if not object_path.exists():
+            raise FileNotFoundError(f"NoSuchKey: {Key}")
+
+        async with self._lock:
+            body = await asyncio.to_thread(object_path.read_bytes)
+            metadata = await self._read_metadata(meta_path)
+
+        # Get file stats
+        stat_info = await asyncio.to_thread(object_path.stat)
+        
+        return {
+            "Body": body,
+            "ContentType": metadata.get("content_type", "application/octet-stream"),
+            "Metadata": metadata.get("metadata", {}),
+            "ContentLength": len(body),
+            "LastModified": datetime.fromtimestamp(stat_info.st_mtime),
+            "ETag": metadata.get("etag", ""),
+        }
+
+    async def head_object(
+        self, 
+        *, 
+        Bucket: str,  # noqa: N803
+        Key: str      # noqa: N803
+    ):
+        """Get object metadata without body."""
+        if self._closed:
+            raise RuntimeError("Client has been closed")
+
+        object_path = self._get_object_path(Bucket, Key)
+        meta_path = self._get_metadata_path(object_path)
+
+        if not object_path.exists():
+            raise FileNotFoundError(f"NoSuchKey: {Key}")
+
+        async with self._lock:
+            metadata = await self._read_metadata(meta_path)
+            stat_info = await asyncio.to_thread(object_path.stat)
+
+        return {
+            "ContentType": metadata.get("content_type", "application/octet-stream"),
+            "Metadata": metadata.get("metadata", {}),
+            "ContentLength": stat_info.st_size,
+            "LastModified": datetime.fromtimestamp(stat_info.st_mtime),
+            "ETag": metadata.get("etag", ""),
+        }
+
+    async def head_bucket(self, *, Bucket: str):  # noqa: N803
+        """Check if bucket (directory) exists."""
+        if self._closed:
+            raise RuntimeError("Client has been closed")
+            
+        bucket_path = self._root / Bucket
+        if not bucket_path.exists():
+            raise FileNotFoundError(f"NoSuchBucket: {Bucket}")
+            
+        return {"ResponseMetadata": {"HTTPStatusCode": 200}}
+
+    async def generate_presigned_url(
+        self, 
+        operation: str, 
+        *, 
+        Params: Dict[str, str],   # noqa: N803
+        ExpiresIn: int           # noqa: N803
+    ) -> str:
+        """
+        Generate file:// URLs for filesystem objects.
+        
+        Note: file:// URLs don't have real expiration, but we include
+        expiry info for compatibility.
+        """
+        if self._closed:
+            raise RuntimeError("Client has been closed")
+
+        bucket, key = Params["Bucket"], Params["Key"]
+        object_path = self._get_object_path(bucket, key)
+
+        if not object_path.exists():
+            raise FileNotFoundError(f"Object not found: {bucket}/{key}")
+
+        # Create file:// URL with query parameters for compatibility
+        return (
+            f"file://{object_path.as_posix()}"
+            f"?operation={operation}"
+            f"&token={uuid.uuid4().hex}"
+            f"&expires={int(time.time()) + ExpiresIn}"
+        )
+
+    async def list_objects_v2(
+        self, 
+        *, 
+        Bucket: str,                    # noqa: N803
+        Prefix: str = "",               # noqa: N803
+        MaxKeys: int = 1000,            # noqa: N803
+        ContinuationToken: Optional[str] = None  # noqa: N803
+    ):
+        """List objects in bucket with optional prefix filtering."""
+        if self._closed:
+            raise RuntimeError("Client has been closed")
+
+        bucket_path = self._root / Bucket
+        if not bucket_path.exists():
+            return {
+                "Contents": [],
+                "KeyCount": 0,
+                "IsTruncated": False,
+            }
+
+        contents = []
+        total_found = 0
+        
+        async with self._lock:
+            # Walk the directory tree
+            for item in bucket_path.rglob("*"):
+                if item.is_file() and not item.name.endswith(".meta.json"):
+                    # Get relative path from bucket root as the key
+                    relative_path = item.relative_to(bucket_path)
+                    key = relative_path.as_posix()
+                    
+                    # Apply prefix filter
+                    if not key.startswith(Prefix):
+                        continue
+                    
+                    total_found += 1
+                    
+                    # Apply pagination
+                    if len(contents) >= MaxKeys:
+                        break
+                    
+                    # Get file stats and metadata
+                    stat_info = await asyncio.to_thread(item.stat)
+                    meta_path = self._get_metadata_path(item)
+                    metadata = await self._read_metadata(meta_path)
+                    
+                    contents.append({
+                        "Key": key,
+                        "Size": stat_info.st_size,
+                        "LastModified": datetime.fromtimestamp(stat_info.st_mtime),
+                        "ETag": f'"{metadata.get("etag", "")}"',
+                        "StorageClass": "STANDARD"
+                    })
+
+        return {
+            "Contents": contents,
+            "KeyCount": len(contents),
+            "IsTruncated": total_found > MaxKeys,
+            "MaxKeys": MaxKeys,
+            "Prefix": Prefix,
+        }
+
+    async def delete_object(
+        self, 
+        *, 
+        Bucket: str,  # noqa: N803
+        Key: str      # noqa: N803
+    ):
+        """Delete object and its metadata from filesystem."""
+        if self._closed:
+            raise RuntimeError("Client has been closed")
+
+        object_path = self._get_object_path(Bucket, Key)
+        meta_path = self._get_metadata_path(object_path)
+
+        async with self._lock:
+            # Remove object file
+            try:
+                await asyncio.to_thread(object_path.unlink)
+            except FileNotFoundError:
+                pass  # S3 doesn't error if object doesn't exist
+            
+            # Remove metadata file
+            try:
+                await asyncio.to_thread(meta_path.unlink)
+            except FileNotFoundError:
+                pass
+            
+            # Clean up empty directories
+            try:
+                await asyncio.to_thread(object_path.parent.rmdir)
+            except OSError:
+                pass  # Directory not empty or other issue
+
+        return {"ResponseMetadata": {"HTTPStatusCode": 204}}
+
+    async def delete_objects(
+        self, 
+        *, 
+        Bucket: str,                           # noqa: N803
+        Delete: Dict[str, List[Dict[str, str]]]  # noqa: N803
+    ):
+        """Delete multiple objects (batch operation)."""
+        if self._closed:
+            raise RuntimeError("Client has been closed")
+
+        deleted = []
+        errors = []
+
+        for obj in Delete.get("Objects", []):
+            key = obj["Key"]
+            try:
+                await self.delete_object(Bucket=Bucket, Key=key)
+                deleted.append({"Key": key})
+            except Exception as e:
+                errors.append({
+                    "Key": key,
+                    "Code": "InternalError",
+                    "Message": str(e)
+                })
+
+        return {
+            "Deleted": deleted,
+            "Errors": errors,
+        }
+
+    async def copy_object(
+        self,
+        *,
+        Bucket: str,              # noqa: N803
+        Key: str,                 # noqa: N803
+        CopySource: Dict[str, str]  # noqa: N803
+    ):
+        """Copy object within filesystem."""
+        if self._closed:
+            raise RuntimeError("Client has been closed")
+
+        source_bucket = CopySource["Bucket"]
+        source_key = CopySource["Key"]
+        
+        # Read source object
+        source_obj = await self.get_object(Bucket=source_bucket, Key=source_key)
+        
+        # Write to destination
+        result = await self.put_object(
+            Bucket=Bucket,
+            Key=Key,
+            Body=source_obj["Body"],
+            ContentType=source_obj["ContentType"],
+            Metadata=source_obj["Metadata"]
+        )
+        
+        return {
+            "CopyObjectResult": {
+                "ETag": result["ETag"],
+                "LastModified": datetime.utcnow()
+            }
+        }
+
+    async def close(self):
+        """Mark client as closed."""
+        self._closed = True
+
+    # ------------------------------------------------------------
+    # Utility/debugging methods
+    # ------------------------------------------------------------
+    
+    async def _debug_get_stats(self) -> Dict[str, Any]:
+        """Get storage statistics."""
+        if self._closed:
+            return {"error": "Client closed"}
+            
+        total_objects = 0
+        total_bytes = 0
+        
+        for item in self._root.rglob("*"):
+            if item.is_file() and not item.name.endswith(".meta.json"):
+                total_objects += 1
+                total_bytes += item.stat().st_size
+                
+        return {
+            "root_path": str(self._root),
+            "total_objects": total_objects,
+            "total_bytes": total_bytes,
+            "closed": self._closed,
+        }
+
+    async def _debug_cleanup_empty_dirs(self):
+        """Remove empty directories (cleanup utility)."""
+        async with self._lock:
+            for item in reversed(sorted(self._root.rglob("*"))):
+                if item.is_dir():
+                    try:
+                        await asyncio.to_thread(item.rmdir)
+                    except OSError:
+                        pass  # Directory not empty
+
+
+# ---- public factory -------------------------------------------------------
+
+def factory(root: Optional[Path] = None) -> Callable[[], AsyncContextManager]:
+    """
+    Create a filesystem client factory.
+    
+    Parameters
+    ----------
+    root : Path, optional
+        Root directory for storage. If None, uses $ARTIFACT_FS_ROOT or ./artifacts
+    """
+    if root is None:
+        root = _ROOT
+    else:
+        root = Path(root).expanduser().resolve()
+        root.mkdir(parents=True, exist_ok=True)
+
+    @asynccontextmanager
+    async def _ctx():
+        client = _FilesystemClient(root)
+        try:
+            yield client
+        finally:
+            await client.close()
+
+    return _ctx
+
+
+# ---- convenience functions ------------------------------------------------
+
+def create_temp_filesystem_factory() -> tuple[Callable[[], AsyncContextManager], Path]:
+    """
+    Create a factory using a temporary directory.
+    
+    Returns
+    -------
+    tuple
+        (factory_function, temp_directory_path)
+    """
+    import tempfile
+    temp_dir = Path(tempfile.mkdtemp(prefix="artifacts_"))
+    return factory(temp_dir), temp_dir
+
+
+async def cleanup_filesystem_store(root: Path):
+    """
+    Clean up a filesystem store directory.
+    
+    Parameters
+    ----------
+    root : Path
+        Directory to clean up
+    """
+    import shutil
+    if root.exists():
+        await asyncio.to_thread(shutil.rmtree, root)

chuk_artifacts/providers/ibm_cos.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+# chuk_artifacts/providers/ibm_cos.py
+"""
+Factory for an aioboto3 client wired for IBM Cloud Object Storage (COS).
+Supports both IAM and HMAC auth.
+
+aioboto3 ≥ 12 returns an *async-context* client, so we expose
+    • factory() - preferred, used by provider_factory
+    • client()  - retained for backward-compat tests/manual use
+"""
+
+from __future__ import annotations
+import os, aioboto3
+from aioboto3.session import AioConfig  # ✅ CRITICAL: Import AioConfig
+from typing import Optional, Callable, AsyncContextManager
+
+# ──────────────────────────────────────────────────────────────────
+# internal helper that actually builds the client
+# ──────────────────────────────────────────────────────────────────
+def _build_client(
+    *,
+    endpoint_url: str,
+    region: str,
+    ibm_api_key: Optional[str],
+    ibm_instance_crn: Optional[str],
+    access_key: Optional[str],
+    secret_key: Optional[str],
+):
+    session = aioboto3.Session()
+
+    # IAM auth (preferred)
+    if not access_key and not secret_key:
+        return session.client(
+            "s3",
+            endpoint_url=endpoint_url,
+            region_name=region,
+            ibm_api_key_id=ibm_api_key,
+            ibm_service_instance_id=ibm_instance_crn,
+            # ✅ Use SigV2 for IBM COS IAM + path style
+            config=AioConfig(
+                signature_version='s3',
+                s3={'addressing_style': 'path'}
+            )
+        )
+
+    # HMAC auth
+    return session.client(
+        "s3",
+        endpoint_url=endpoint_url,
+        region_name=region,
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        # ✅ Use SigV2 for IBM COS HMAC + path style
+        config=AioConfig(
+            signature_version='s3',
+            s3={'addressing_style': 'path'}
+        )
+    )
+
+
+# ──────────────────────────────────────────────────────────────────
+# public factory  (provider_factory expects this)
+# ──────────────────────────────────────────────────────────────────
+def factory(
+    *,
+    endpoint_url: Optional[str] = None,
+    region: str = "us-south",
+    access_key: Optional[str] = None,
+    secret_key: Optional[str] = None,
+):
+    """
+    Return an async-context S3 client for IBM COS (HMAC only).
+    """
+    endpoint_url = endpoint_url or os.getenv(
+        "IBM_COS_ENDPOINT",
+        "https://s3.us-south.cloud-object-storage.appdomain.cloud",
+    )
+    access_key = access_key or os.getenv("AWS_ACCESS_KEY_ID")
+    secret_key = secret_key or os.getenv("AWS_SECRET_ACCESS_KEY")
+    
+    # ✅ Extract region from endpoint to ensure they match
+    if endpoint_url:
+        if "us-south" in endpoint_url:
+            region = "us-south"
+        elif "us-east" in endpoint_url:
+            region = "us-east-1"
+        elif "eu-gb" in endpoint_url:
+            region = "eu-gb"
+        elif "eu-de" in endpoint_url:
+            region = "eu-de"
+    
+    # Check AWS_REGION environment variable as override
+    env_region = os.getenv('AWS_REGION')
+    if env_region:
+        region = env_region
+
+    if not (access_key and secret_key):
+        raise RuntimeError(
+            "HMAC credentials missing. "
+            "Set AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY "
+            "or generate an HMAC key for your COS instance."
+        )
+
+    def _make() -> AsyncContextManager:
+        session = aioboto3.Session()
+        return session.client(
+            "s3",
+            endpoint_url=endpoint_url,
+            region_name=region,
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key,
+            # ✅ CRITICAL: IBM COS requires Signature Version 2 for writes AND presigned URLs
+            config=AioConfig(
+                signature_version='s3',
+                s3={
+                    'addressing_style': 'path'  # Also ensure path-style addressing
+                }
+            )
+        )
+
+    return _make

chuk_artifacts/providers/ibm_cos_iam.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+# chuk_artifacts/providers/ibm_cos_iam.py
+"""
+Async wrapper for IBM Cloud Object Storage using IAM API-key (oauth).
+
+✓ Fits the aioboto3-style interface that ArtifactStore expects:
+    • async put_object(...)
+    • async generate_presigned_url(...)
+✓ No HMAC keys required - just IBM_COS_APIKEY + IBM_COS_INSTANCE_CRN.
+
+Env vars
+--------
+IBM_COS_APIKEY           - value of "apikey" field
+IBM_COS_INSTANCE_CRN     - value of "resource_instance_id"
+IBM_COS_ENDPOINT         - regional data endpoint, e.g.
+                           https://s3.us-south.cloud-object-storage.appdomain.cloud
+"""
+
+from __future__ import annotations
+import os, asyncio
+from contextlib import asynccontextmanager
+from typing import AsyncContextManager, Any, Dict, Callable
+
+import ibm_boto3
+from ibm_botocore.client import Config
+
+
+# ─────────────────────────────────────────────────────────────────────
+def _sync_client():
+    endpoint = os.getenv(
+        "IBM_COS_ENDPOINT",
+        "https://s3.us-south.cloud-object-storage.appdomain.cloud",
+    )
+    api_key = os.getenv("IBM_COS_APIKEY")
+    instance = os.getenv("IBM_COS_INSTANCE_CRN")
+    if not (api_key and instance):
+        raise RuntimeError(
+            "Set IBM_COS_APIKEY, IBM_COS_INSTANCE_CRN, IBM_COS_ENDPOINT "
+            "for ibm_cos_iam provider."
+        )
+    return ibm_boto3.client(
+        "s3",
+        ibm_api_key_id=api_key,
+        ibm_service_instance_id=instance,
+        config=Config(signature_version="oauth"),
+        endpoint_url=endpoint,
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────
+class _AsyncIBMClient:
+    """Minimal async façade over synchronous ibm_boto3 S3 client."""
+    def __init__(self, sync_client):
+        self._c = sync_client
+
+    # ---- methods used by ArtifactStore -------------------------------------
+    async def put_object(self, **kw) -> Dict[str, Any]:
+        return await asyncio.to_thread(self._c.put_object, **kw)
+
+    async def generate_presigned_url(self, *a, **kw) -> str:
+        return await asyncio.to_thread(self._c.generate_presigned_url, *a, **kw)
+
+    # ---- cleanup -----------------------------------------------------------
+    async def close(self):
+        await asyncio.to_thread(self._c.close)
+
+
+# ─────────────────────────────────────────────────────────────────────
+def factory() -> Callable[[], AsyncContextManager]:
+    """
+    Return a zero-arg callable that yields an async-context-manager.
+    """
+
+    @asynccontextmanager
+    async def _ctx():
+        sync_client = _sync_client()
+        try:
+            yield _AsyncIBMClient(sync_client)
+        finally:
+            await asyncio.to_thread(sync_client.close)
+
+    return _ctx  # Return the function, not the result of calling it