chunksmith-adapters 0.3.0__tar.gz

chunksmith_adapters-0.3.0/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.4
+Name: chunksmith-adapters
+Version: 0.3.0
+Summary: Optional ChunkSmith storage adapters (S3, MongoDB, PostgreSQL, MVL composite).
+Author-email: AnshulParate2004 <anshulnparate@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/AnshulParate2004/ChunkSmith
+Project-URL: Repository, https://github.com/AnshulParate2004/ChunkSmith
+Project-URL: Changelog, https://github.com/AnshulParate2004/ChunkSmith/blob/main/CHANGELOG.md
+Requires-Python: >=3.10
+Requires-Dist: chunksmith-core>=0.3.0
+Provides-Extra: s3
+Requires-Dist: boto3>=1.35.0; extra == "s3"
+Provides-Extra: mongo
+Requires-Dist: pymongo>=4.10.0; extra == "mongo"
+Provides-Extra: postgres
+Requires-Dist: psycopg[binary]>=3.2.0; extra == "postgres"
+Provides-Extra: mvl
+Requires-Dist: boto3>=1.35.0; extra == "mvl"
+Requires-Dist: pymongo>=4.10.0; extra == "mvl"
+Requires-Dist: psycopg[binary]>=3.2.0; extra == "mvl"

chunksmith_adapters-0.3.0/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "chunksmith-adapters"
+version = "0.3.0"
+description = "Optional ChunkSmith storage adapters (S3, MongoDB, PostgreSQL, MVL composite)."
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "AnshulParate2004", email = "anshulnparate@gmail.com" }]
+dependencies = [
+    "chunksmith-core>=0.3.0",
+]
+
+[project.optional-dependencies]
+s3 = ["boto3>=1.35.0"]
+mongo = ["pymongo>=4.10.0"]
+postgres = ["psycopg[binary]>=3.2.0"]
+mvl = [
+    "boto3>=1.35.0",
+    "pymongo>=4.10.0",
+    "psycopg[binary]>=3.2.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/AnshulParate2004/ChunkSmith"
+Repository = "https://github.com/AnshulParate2004/ChunkSmith"
+Changelog = "https://github.com/AnshulParate2004/ChunkSmith/blob/main/CHANGELOG.md"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["chunksmith_adapters", "chunksmith_adapters.*"]

chunksmith_adapters-0.3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

chunksmith_adapters-0.3.0/src/chunksmith_adapters/__init__.py

@@ -0,0 +1,40 @@
+"""Optional storage adapters for ChunkSmith pipelines."""
+
+from chunksmith_adapters.base import BasePipelineStorage, ProgressRecord
+from chunksmith_adapters.config import (
+    FilesystemStorageConfig,
+    MongoStorageConfig,
+    S3StorageConfig,
+    StorageConfig,
+    build_storage,
+)
+from chunksmith_adapters.filesystem import FilesystemArtifactSink, FilesystemStorage
+from chunksmith_adapters.mvl_composite import MvlCompositeStorage, MvlStorageConfig
+
+__all__ = [
+    "BasePipelineStorage",
+    "FilesystemArtifactSink",
+    "FilesystemStorage",
+    "FilesystemStorageConfig",
+    "MongoStorageConfig",
+    "MvlCompositeStorage",
+    "MvlStorageConfig",
+    "ProgressRecord",
+    "S3StorageConfig",
+    "StorageConfig",
+    "build_storage",
+]
+
+try:
+    from chunksmith_adapters.s3 import S3Storage
+
+    __all__.append("S3Storage")
+except ImportError:
+    S3Storage = None  # type: ignore[misc, assignment]
+
+try:
+    from chunksmith_adapters.mongodb import MongoStorage
+
+    __all__.append("MongoStorage")
+except ImportError:
+    MongoStorage = None  # type: ignore[misc, assignment]

chunksmith_adapters-0.3.0/src/chunksmith_adapters/base.py

@@ -0,0 +1,78 @@
+"""Shared progressive storage base."""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ProgressRecord:
+    event: str
+    payload: dict[str, Any]
+    at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+
+
+class BasePipelineStorage:
+    """
+    Base class for progressive storage adapters.
+
+    Subclasses implement ``_write_blob`` / ``_write_json``; this class handles
+    progress tracking and artifact serialization.
+    """
+
+    def __init__(self, *, document_id: str, run_id: str | None = None) -> None:
+        self.document_id = document_id
+        self.run_id = run_id or datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        self.progress_log: list[ProgressRecord] = []
+        self._meta: dict[str, Any] = {
+            "document_id": document_id,
+            "run_id": self.run_id,
+            "status": "running",
+            "updated_at": None,
+        }
+
+    def on_progress(self, event: str, payload: dict[str, Any]) -> None:
+        self.progress_log.append(ProgressRecord(event=event, payload=dict(payload)))
+        self._meta["last_event"] = event
+        self._meta["updated_at"] = datetime.now(timezone.utc).isoformat()
+        self._persist_progress()
+        logger.debug("[chunksmith storage] %s document_id=%s %s", event, self.document_id, payload)
+
+    def save_elements(self, elements: list[dict[str, Any]]) -> None:
+        self._write_json("elements.json", elements)
+        self._meta["element_count"] = len(elements)
+
+    def save_coded_formate(self, text: str) -> None:
+        self._write_blob("coded_formate.txt", text.encode("utf-8"), "text/plain")
+
+    def save_outline(self, outline: dict[str, Any]) -> None:
+        self._write_json("outline.json", outline)
+
+    def save_mapper(self, mapper: dict[str, Any]) -> None:
+        self._write_json("mapper.json", mapper)
+
+    def finalize(self) -> None:
+        self._meta["status"] = "completed"
+        self._meta["updated_at"] = datetime.now(timezone.utc).isoformat()
+        self._write_json("meta.json", self._meta)
+        self._write_json("progress.json", [r.__dict__ for r in self.progress_log])
+
+    def _persist_progress(self) -> None:
+        self._write_json("progress.json", [r.__dict__ for r in self.progress_log])
+        self._write_json("meta.json", self._meta)
+
+    def _write_json(self, name: str, data: Any) -> None:
+        raw = json.dumps(data, ensure_ascii=False, indent=2).encode("utf-8")
+        self._write_blob(name, raw, "application/json")
+
+    def _write_blob(self, name: str, data: bytes, content_type: str) -> None:
+        raise NotImplementedError
+
+    def _key(self, name: str) -> str:
+        return f"{self.document_id}/{self.run_id}/{name}"

chunksmith_adapters-0.3.0/src/chunksmith_adapters/config.py

@@ -0,0 +1,96 @@
+"""Class-based storage configuration — no environment variables."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Union
+
+from chunksmith_core.ports import PipelineStorage
+
+
+@dataclass
+class FilesystemStorageConfig:
+    """Local disk storage under ``{root}/{document_id}/{run_id}/``."""
+
+    root: str = "./chunksmith-runs"
+    document_id: str = "default"
+
+    def build(self) -> PipelineStorage:
+        from chunksmith_adapters.filesystem import FilesystemStorage
+
+        return FilesystemStorage(self.root, document_id=self.document_id)
+
+
+@dataclass
+class S3StorageConfig:
+    """Generic S3 progressive storage (simple prefix layout)."""
+
+    bucket: str
+    document_id: str
+    prefix: str = "chunksmith/"
+    region: str | None = None
+    access_key_id: str = ""
+    secret_access_key: str = ""
+    endpoint_url: str | None = None
+
+    def build(self) -> PipelineStorage:
+        from chunksmith_adapters.s3 import S3Storage
+
+        client = None
+        if self.access_key_id and self.secret_access_key:
+            try:
+                import boto3
+
+                kwargs = {
+                    "aws_access_key_id": self.access_key_id,
+                    "aws_secret_access_key": self.secret_access_key,
+                }
+                if self.endpoint_url:
+                    kwargs["endpoint_url"] = self.endpoint_url
+                if self.region:
+                    kwargs["region_name"] = self.region
+                client = boto3.client("s3", **kwargs)
+            except ImportError as exc:
+                raise ImportError(
+                    "S3Storage requires boto3. Install with: pip install 'chunksmith-adapters[s3]'"
+                ) from exc
+
+        return S3Storage(
+            bucket=self.bucket,
+            prefix=self.prefix,
+            document_id=self.document_id,
+            region=self.region,
+            client=client,
+        )
+
+
+@dataclass
+class MongoStorageConfig:
+    """Generic Mongo progressive storage (one document per run)."""
+
+    uri: str
+    document_id: str
+    database: str = "chunksmith"
+    collection: str = "indexes"
+
+    def build(self) -> PipelineStorage:
+        from chunksmith_adapters.mongodb import MongoStorage
+
+        return MongoStorage(
+            uri=self.uri,
+            database=self.database,
+            collection=self.collection,
+            document_id=self.document_id,
+        )
+
+
+StorageConfig = Union[
+    "FilesystemStorageConfig",
+    "S3StorageConfig",
+    "MongoStorageConfig",
+]
+
+
+def build_storage(config: StorageConfig) -> PipelineStorage:
+    """Build a storage adapter from a config dataclass."""
+    return config.build()

chunksmith_adapters-0.3.0/src/chunksmith_adapters/filesystem.py

@@ -0,0 +1,50 @@
+"""Local filesystem progressive storage."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from chunksmith_adapters.base import BasePipelineStorage
+
+
+class FilesystemArtifactSink:
+    """Write indexing artifacts to a local directory (post-run via ``persist_*`` helpers)."""
+
+    def __init__(self, root: str | Path) -> None:
+        self.root = Path(root)
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def save_elements(self, elements: list[dict[str, Any]]) -> Path:
+        path = self.root / "elements.json"
+        path.write_text(json.dumps(elements, ensure_ascii=False, indent=2), encoding="utf-8")
+        return path
+
+    def save_coded_formate(self, text: str) -> Path:
+        path = self.root / "coded_formate.txt"
+        path.write_text(text, encoding="utf-8")
+        return path
+
+    def save_outline(self, outline: dict[str, Any]) -> Path:
+        path = self.root / "outline.json"
+        path.write_text(json.dumps(outline, ensure_ascii=False, indent=2), encoding="utf-8")
+        return path
+
+    def save_mapper(self, mapper: dict[str, Any]) -> Path:
+        path = self.root / "mapper.json"
+        path.write_text(json.dumps(mapper, ensure_ascii=False, indent=2), encoding="utf-8")
+        return path
+
+
+class FilesystemStorage(BasePipelineStorage):
+    """Save artifacts under ``{root}/{document_id}/{run_id}/`` as the pipeline runs."""
+
+    def __init__(self, root: str | Path, *, document_id: str) -> None:
+        super().__init__(document_id=document_id)
+        self.root = Path(root)
+
+    def _write_blob(self, name: str, data: bytes, content_type: str) -> None:
+        path = self.root / self._key(name)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(data)

chunksmith_adapters-0.3.0/src/chunksmith_adapters/mongodb.py

@@ -0,0 +1,70 @@
+"""MongoDB progressive storage (optional pymongo)."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from chunksmith_adapters.base import BasePipelineStorage
+
+
+class MongoStorage(BasePipelineStorage):
+    """
+    Upsert one document per run; updates fields as pipeline stages complete.
+
+    Requires: ``pip install 'chunksmith-adapters[mongo]'`` or ``pip install pymongo``
+    """
+
+    def __init__(
+        self,
+        *,
+        uri: str,
+        database: str,
+        collection: str,
+        document_id: str,
+        client: Any | None = None,
+    ) -> None:
+        if not uri:
+            raise ValueError("MongoStorage requires uri=")
+        super().__init__(document_id=document_id)
+        self._db_name = database
+        self._collection_name = collection
+        self._client = client or self._make_client(uri)
+        self._collection = self._client[database][collection]
+        self._doc_key = {"document_id": document_id, "run_id": self.run_id}
+        self._collection.update_one(self._doc_key, {"$set": {**self._doc_key, "status": "running"}}, upsert=True)
+
+    @staticmethod
+    def _make_client(uri: str):
+        try:
+            from pymongo import MongoClient
+        except ImportError as exc:
+            raise ImportError(
+                "MongoStorage requires pymongo. Install with: pip install 'chunksmith-adapters[mongo]'"
+            ) from exc
+        return MongoClient(uri)
+
+    def _write_blob(self, name: str, data: bytes, content_type: str) -> None:
+        field = name.replace(".", "_")
+        if content_type == "application/json":
+            value: Any = json.loads(data.decode("utf-8"))
+        else:
+            value = data.decode("utf-8")
+        self._collection.update_one(
+            self._doc_key,
+            {"$set": {field: value, "updated_at": self._meta.get("updated_at"), "status": "running"}},
+            upsert=True,
+        )
+
+    def finalize(self) -> None:
+        super().finalize()
+        self._collection.update_one(
+            self._doc_key,
+            {
+                "$set": {
+                    "status": "completed",
+                    "meta": self._meta,
+                    "progress": [r.__dict__ for r in self.progress_log],
+                }
+            },
+        )

chunksmith_adapters-0.3.0/src/chunksmith_adapters/mvl_composite.py

@@ -0,0 +1,173 @@
+"""Unified MVL storage: S3 + MongoDB + PostgreSQL with graceful partial config."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from chunksmith_adapters.mvl_mongo import MvlMongoStorage
+from chunksmith_adapters.mvl_postgres import MvlPostgresStorage
+from chunksmith_adapters.mvl_s3 import MvlS3Storage
+
+logger = logging.getLogger(__name__)
+
+ContextFormat = Literal["json", "toon"]
+
+
+@dataclass
+class MvlStorageConfig:
+    """
+    One config for all MVL backends. Set only what you have — missing backends log a warning and are skipped.
+
+    Example: S3 + Postgres configured, Mongo omitted → stores to S3 and Postgres only.
+    """
+
+    user_id: str
+    project_id: str
+    document_id: str
+    mapping_method: str = "page_indexing"
+    context_format: ContextFormat = "json"
+    indexing_mode: str = "chunksmith_multi_indexing"
+
+    # --- S3 (optional) ---
+    s3_bucket: str | None = None
+    s3_access_key_id: str = ""
+    s3_secret_access_key: str = ""
+    s3_endpoint_url: str | None = None
+    s3_region: str | None = None
+
+    # --- MongoDB (optional) ---
+    mongo_uri: str | None = None
+    mongo_database: str = "chunksmith"
+    mongo_collection: str = "document_indexes"
+
+    # --- PostgreSQL (optional) ---
+    postgres_url: str | None = None
+
+    warnings: list[str] = field(default_factory=list, init=False, repr=False)
+    active_backends: list[str] = field(default_factory=list, init=False, repr=False)
+
+    def build(self) -> "MvlCompositeStorage":
+        return MvlCompositeStorage.from_config(self)
+
+
+class MvlCompositeStorage:
+    """Fan-out storage to every configured MVL backend; warn on missing configs."""
+
+    def __init__(
+        self,
+        *,
+        s3: MvlS3Storage | None,
+        mongo: MvlMongoStorage | None,
+        postgres: MvlPostgresStorage | None,
+        warnings: list[str],
+        active_backends: list[str],
+    ) -> None:
+        self._s3 = s3
+        self._mongo = mongo
+        self._postgres = postgres
+        self.warnings = warnings
+        self.active_backends = active_backends
+
+    @classmethod
+    def from_config(cls, config: MvlStorageConfig) -> "MvlCompositeStorage":
+        warnings: list[str] = []
+        active: list[str] = []
+        s3: MvlS3Storage | None = None
+        mongo: MvlMongoStorage | None = None
+        postgres: MvlPostgresStorage | None = None
+
+        if config.s3_bucket:
+            s3 = MvlS3Storage(
+                user_id=config.user_id,
+                project_id=config.project_id,
+                document_id=config.document_id,
+                bucket=config.s3_bucket,
+                mapping_method=config.mapping_method,
+                context_format=config.context_format,
+                access_key_id=config.s3_access_key_id,
+                secret_access_key=config.s3_secret_access_key,
+                endpoint_url=config.s3_endpoint_url,
+                region=config.s3_region,
+            )
+            active.append("S3")
+        else:
+            warnings.append("S3 not configured — skipping S3 storage.")
+
+        if config.mongo_uri:
+            mongo = MvlMongoStorage(
+                uri=config.mongo_uri,
+                user_id=config.user_id,
+                project_id=config.project_id,
+                document_id=config.document_id,
+                database=config.mongo_database,
+                collection=config.mongo_collection,
+            )
+            active.append("MongoDB")
+        else:
+            warnings.append("MongoDB not configured — skipping MongoDB storage.")
+
+        if config.postgres_url:
+            postgres = MvlPostgresStorage(
+                database_url=config.postgres_url,
+                user_id=config.user_id,
+                project_id=config.project_id,
+                document_id=config.document_id,
+                indexing_mode=config.indexing_mode,
+            )
+            active.append("PostgreSQL")
+        else:
+            warnings.append("PostgreSQL not configured — skipping PostgreSQL storage.")
+
+        if not active:
+            raise ValueError("No storage backend configured. Set at least one of: s3_bucket, mongo_uri, postgres_url.")
+
+        config.warnings = list(warnings)
+        config.active_backends = list(active)
+
+        for msg in warnings:
+            logger.warning("%s Will store using: %s.", msg, ", ".join(active))
+
+        logger.info(
+            "MVL composite storage active backends: %s (document_id=%s)",
+            ", ".join(active),
+            config.document_id,
+        )
+
+        return cls(s3=s3, mongo=mongo, postgres=postgres, warnings=warnings, active_backends=active)
+
+    def _call(self, method: str, *args: Any) -> None:
+        for name, backend in (
+            ("S3", self._s3),
+            ("MongoDB", self._mongo),
+            ("PostgreSQL", self._postgres),
+        ):
+            if backend is None:
+                continue
+            fn = getattr(backend, method, None)
+            if fn is None:
+                continue
+            try:
+                fn(*args)
+            except Exception as exc:
+                logger.error("%s storage.%s failed: %s", name, method, exc)
+                raise
+
+    def on_progress(self, event: str, payload: dict[str, Any]) -> None:
+        self._call("on_progress", event, payload)
+
+    def save_elements(self, elements: list[dict[str, Any]]) -> None:
+        self._call("save_elements", elements)
+
+    def save_coded_formate(self, text: str) -> None:
+        self._call("save_coded_formate", text)
+
+    def save_outline(self, outline: dict[str, Any]) -> None:
+        self._call("save_outline", outline)
+
+    def save_mapper(self, mapper: dict[str, Any]) -> None:
+        self._call("save_mapper", mapper)
+
+    def finalize(self) -> None:
+        self._call("finalize")

chunksmith_adapters-0.3.0/src/chunksmith_adapters/mvl_mongo.py

@@ -0,0 +1,135 @@
+"""MVL Backend-style MongoDB storage (scoped by user_id / project_id / document_id)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+from chunksmith_adapters.base import ProgressRecord
+
+
+@dataclass
+class MvlMongoStorageConfig:
+    """
+    Connect and store like MVL ``MongoDocumentStore`` document_indexes layout.
+
+    One document per run in ``document_indexes`` collection, updated as pipeline progresses.
+    """
+
+    uri: str
+    user_id: str
+    project_id: str
+    document_id: str
+    database: str = "chunksmith"
+    collection: str = "document_indexes"
+
+    def build(self) -> "MvlMongoStorage":
+        return MvlMongoStorage(
+            uri=self.uri,
+            user_id=self.user_id,
+            project_id=self.project_id,
+            document_id=self.document_id,
+            database=self.database,
+            collection=self.collection,
+        )
+
+
+class MvlMongoStorage:
+    """Progressive Mongo upserts — same scope fields as MVL Backend indexing context."""
+
+    def __init__(
+        self,
+        *,
+        uri: str,
+        user_id: str,
+        project_id: str,
+        document_id: str,
+        database: str = "chunksmith",
+        collection: str = "document_indexes",
+        client: Any | None = None,
+    ) -> None:
+        if not uri:
+            raise ValueError("MvlMongoStorage requires uri=")
+        self.user_id = user_id
+        self.project_id = project_id
+        self.document_id = document_id
+        self.run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        self.progress_log: list[ProgressRecord] = []
+        self._client = client or self._make_client(uri)
+        self._collection = self._client[database][collection]
+        self._filter = {
+            "user_id": user_id,
+            "project_id": project_id,
+            "document_id": document_id,
+            "run_id": self.run_id,
+        }
+        self._collection.update_one(
+            self._filter,
+            {
+                "$set": {
+                    **self._filter,
+                    "status": "running",
+                    "updated_at": datetime.now(timezone.utc).isoformat(),
+                }
+            },
+            upsert=True,
+        )
+
+    @staticmethod
+    def _make_client(uri: str):
+        try:
+            from pymongo import MongoClient
+        except ImportError as exc:
+            raise ImportError(
+                "MvlMongoStorage requires pymongo. Install with: pip install 'chunksmith-adapters[mongo]'"
+            ) from exc
+        return MongoClient(uri)
+
+    def on_progress(self, event: str, payload: dict[str, Any]) -> None:
+        self.progress_log.append(ProgressRecord(event=event, payload=dict(payload)))
+        self._collection.update_one(
+            self._filter,
+            {
+                "$set": {
+                    "last_event": event,
+                    "progress": [r.__dict__ for r in self.progress_log],
+                    "updated_at": datetime.now(timezone.utc).isoformat(),
+                }
+            },
+        )
+
+    def save_elements(self, elements: list[dict[str, Any]]) -> None:
+        self._set_field("elements", elements)
+
+    def save_coded_formate(self, text: str) -> None:
+        self._set_field("title_coded_formate", text)
+
+    def save_outline(self, outline: dict[str, Any]) -> None:
+        self._set_field("outline", outline)
+
+    def save_mapper(self, mapper: dict[str, Any]) -> None:
+        self._set_field("mapper_output", mapper)
+
+    def finalize(self) -> None:
+        self._collection.update_one(
+            self._filter,
+            {
+                "$set": {
+                    "status": "completed",
+                    "updated_at": datetime.now(timezone.utc).isoformat(),
+                }
+            },
+        )
+
+    def _set_field(self, name: str, value: Any) -> None:
+        self._collection.update_one(
+            self._filter,
+            {
+                "$set": {
+                    name: value,
+                    "updated_at": datetime.now(timezone.utc).isoformat(),
+                    "status": "running",
+                }
+            },
+        )

chunksmith_adapters-0.3.0/src/chunksmith_adapters/mvl_paths.py

@@ -0,0 +1,61 @@
+"""MVL S3 object key layout."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+ContextFormat = Literal["json", "toon"]
+
+_VALID_MAPPING_METHODS = frozenset(
+    {
+        "page_indexing",
+        "title_indexing",
+        "chunk_assignment_indexing",
+        "group_indexing",
+        "anchor_indexing",
+    }
+)
+
+
+def project_prefix(user_id: str, project_id: str) -> str:
+    uid = (user_id or "").strip().strip("/")
+    pid = (project_id or "").strip().strip("/")
+    if not uid or not pid:
+        raise ValueError("user_id and project_id are required for storage paths")
+    return f"{uid}/{pid}"
+
+
+def _scoped(prefix: str, *parts: str) -> str:
+    return "/".join((prefix, *parts))
+
+
+def json_index_object_key(user_id: str, project_id: str, document_id: str) -> str:
+    return _scoped(project_prefix(user_id, project_id), "json", f"{document_id}_index.json")
+
+
+def mapper_json_object_key(
+    document_id: str,
+    mapping_method: str,
+    *,
+    user_id: str,
+    project_id: str,
+    context_format: ContextFormat,
+) -> str:
+    method = (mapping_method or "").strip().lower()
+    if method not in _VALID_MAPPING_METHODS:
+        raise ValueError(f"unknown mapping_method: {mapping_method!r}")
+    fmt = context_format if context_format in ("json", "toon") else "json"
+    basename = f"{document_id}_{method}_{fmt}.json"
+    return _scoped(project_prefix(user_id, project_id), "json", basename)
+
+
+def text_coded_formate_object_key(user_id: str, project_id: str, document_id: str) -> str:
+    return _scoped(project_prefix(user_id, project_id), "text", f"{document_id}_title_coded_formate.txt")
+
+
+def elements_json_object_key(user_id: str, project_id: str, document_id: str) -> str:
+    return _scoped(project_prefix(user_id, project_id), "json", f"{document_id}_elements.json")
+
+
+def progress_json_object_key(user_id: str, project_id: str, document_id: str, run_id: str) -> str:
+    return _scoped(project_prefix(user_id, project_id), "json", f"{document_id}_{run_id}_progress.json")

chunksmith_adapters-0.3.0/src/chunksmith_adapters/mvl_postgres.py

@@ -0,0 +1,162 @@
+"""MVL Backend-style PostgreSQL progress + document status (optional psycopg)."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+from chunksmith_adapters.mvl_progress import progress_tuple
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class MvlPostgresStorageConfig:
+    """PostgreSQL connection for document status + processing_jobs (MVL tables)."""
+
+    database_url: str
+    user_id: str
+    project_id: str
+    document_id: str
+    indexing_mode: str = "chunksmith_multi_indexing"
+
+    def is_configured(self) -> bool:
+        return bool((self.database_url or "").strip())
+
+    def build(self) -> "MvlPostgresStorage":
+        if not self.is_configured():
+            raise ValueError("PostgreSQL database_url is required when building MvlPostgresStorage")
+        return MvlPostgresStorage(
+            database_url=self.database_url,
+            user_id=self.user_id,
+            project_id=self.project_id,
+            document_id=self.document_id,
+            indexing_mode=self.indexing_mode,
+        )
+
+
+class MvlPostgresStorage:
+    """
+    Update MVL ``documents`` and ``processing_jobs`` tables during pipeline progress.
+
+    Does not store artifact blobs (those go to S3 / Mongo).
+    """
+
+    def __init__(
+        self,
+        *,
+        database_url: str,
+        user_id: str,
+        project_id: str,
+        document_id: str,
+        indexing_mode: str = "chunksmith_multi_indexing",
+        conn: Any | None = None,
+    ) -> None:
+        self.database_url = database_url
+        self.user_id = user_id
+        self.project_id = project_id
+        self.document_id = document_id
+        self.indexing_mode = indexing_mode
+        self._conn = conn
+
+    def _connect(self):
+        if self._conn is not None:
+            return self._conn
+        try:
+            import psycopg
+        except ImportError as exc:
+            raise ImportError(
+                "PostgreSQL storage requires psycopg. Install with: pip install 'chunksmith-adapters[postgres]'"
+            ) from exc
+        return psycopg.connect(self.database_url)
+
+    def on_progress(self, event: str, payload: dict[str, Any]) -> None:
+        if event == "llm:token":
+            delta = str(payload.get("delta") or "")
+            if not delta:
+                return
+            step, pct, label = progress_tuple("pipeline:page_index")
+            preview = delta[-200:]
+            message = f"{label} (streaming): …{preview}"
+            now = datetime.now(timezone.utc)
+            sql = """
+                INSERT INTO processing_jobs (document_id, project_id, status, progress, message, step, indexing_mode, updated_at)
+                VALUES (%s, %s, 'running', %s, %s, %s, %s, %s)
+                ON CONFLICT (document_id) DO UPDATE SET
+                    status = EXCLUDED.status,
+                    progress = EXCLUDED.progress,
+                    message = EXCLUDED.message,
+                    step = EXCLUDED.step,
+                    indexing_mode = EXCLUDED.indexing_mode,
+                    updated_at = EXCLUDED.updated_at
+            """
+            with self._connect() as conn:
+                with conn.cursor() as cur:
+                    cur.execute(
+                        sql,
+                        (self.document_id, self.project_id, pct, message, step, self.indexing_mode, now),
+                    )
+                conn.commit()
+            return
+
+        step, pct, label = progress_tuple(event)
+        detail = ", ".join(
+            f"{k}={v}" for k, v in payload.items() if k in {"chunk_count", "elements", "chunks", "nodes"}
+        )
+        message = f"{label}: {detail}" if detail else label
+        now = datetime.now(timezone.utc)
+        sql = """
+            INSERT INTO processing_jobs (document_id, project_id, status, progress, message, step, indexing_mode, updated_at)
+            VALUES (%s, %s, 'running', %s, %s, %s, %s, %s)
+            ON CONFLICT (document_id) DO UPDATE SET
+                status = EXCLUDED.status,
+                progress = EXCLUDED.progress,
+                message = EXCLUDED.message,
+                step = EXCLUDED.step,
+                indexing_mode = EXCLUDED.indexing_mode,
+                updated_at = EXCLUDED.updated_at
+        """
+        with self._connect() as conn:
+            with conn.cursor() as cur:
+                cur.execute(
+                    sql,
+                    (self.document_id, self.project_id, pct, message, step, self.indexing_mode, now),
+                )
+            conn.commit()
+
+    def save_elements(self, elements: list[dict[str, Any]]) -> None:
+        return
+
+    def save_coded_formate(self, text: str) -> None:
+        return
+
+    def save_outline(self, outline: dict[str, Any]) -> None:
+        return
+
+    def save_mapper(self, mapper: dict[str, Any]) -> None:
+        return
+
+    def finalize(self) -> None:
+        now = datetime.now(timezone.utc)
+        with self._connect() as conn:
+            with conn.cursor() as cur:
+                cur.execute(
+                    """
+                    UPDATE processing_jobs
+                    SET status = 'completed', progress = 100, message = 'Indexing completed', updated_at = %s
+                    WHERE document_id = %s
+                    """,
+                    (now, self.document_id),
+                )
+                cur.execute(
+                    """
+                    UPDATE documents
+                    SET status = 'indexed', indexing_mode = %s
+                    WHERE id = %s
+                    """,
+                    (self.indexing_mode, self.document_id),
+                )
+            conn.commit()
+        logger.info("PostgreSQL: document_id=%s marked indexed", self.document_id)

chunksmith_adapters-0.3.0/src/chunksmith_adapters/mvl_progress.py

@@ -0,0 +1,21 @@
+"""Pipeline event → progress step mapping (MVL Backend parity)."""
+
+from __future__ import annotations
+
+PIPELINE_EVENT_PROGRESS: dict[str, tuple[int, int, str]] = {
+    "pipeline:start": (1, 12, "Pipeline started"),
+    "partition:split": (1, 18, "Partitioning PDF pages"),
+    "partition:chunk_done": (1, 24, "Partition chunk completed"),
+    "partition:done": (1, 30, "Partitioning completed"),
+    "pipeline:elements": (2, 40, "Elements loaded"),
+    "pipeline:group_by_title": (3, 50, "Grouping by title"),
+    "pipeline:coded_formate": (4, 60, "Generating coded format"),
+    "pipeline:page_index": (4, 72, "Running page indexing"),
+    "pipeline:mapper": (4, 80, "Building mapper output"),
+    "pipeline:mapper_saved": (4, 82, "Mapper saved"),
+    "pipeline:done": (6, 100, "Indexing completed"),
+}
+
+
+def progress_tuple(event: str) -> tuple[int, int, str]:
+    return PIPELINE_EVENT_PROGRESS.get(event, (1, 10, event))

chunksmith_adapters-0.3.0/src/chunksmith_adapters/mvl_s3.py

@@ -0,0 +1,164 @@
+"""MVL Backend-style S3 storage (scoped keys + boto3 client from config class)."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any, Literal
+
+from chunksmith_adapters.base import ProgressRecord
+from chunksmith_adapters.mvl_paths import (
+    elements_json_object_key,
+    json_index_object_key,
+    mapper_json_object_key,
+    progress_json_object_key,
+    text_coded_formate_object_key,
+)
+
+ContextFormat = Literal["json", "toon"]
+
+
+@dataclass
+class MvlS3StorageConfig:
+    """
+    Connect and store like MVL Backend ``StorageManager`` + ``storage_paths``.
+
+    Keys: ``{user_id}/{project_id}/json/...`` and ``.../text/...`` inside one bucket.
+    """
+
+    user_id: str
+    project_id: str
+    document_id: str
+    bucket: str
+    mapping_method: str = "page_indexing"
+    context_format: ContextFormat = "json"
+
+    access_key_id: str = ""
+    secret_access_key: str = ""
+    endpoint_url: str | None = None
+    region: str | None = None
+
+    def build(self) -> "MvlS3Storage":
+        return MvlS3Storage(
+            user_id=self.user_id,
+            project_id=self.project_id,
+            document_id=self.document_id,
+            bucket=self.bucket,
+            mapping_method=self.mapping_method,
+            context_format=self.context_format,
+            access_key_id=self.access_key_id,
+            secret_access_key=self.secret_access_key,
+            endpoint_url=self.endpoint_url,
+            region=self.region,
+        )
+
+
+class MvlS3Storage:
+    """Progressive S3 uploads using MVL object key layout."""
+
+    def __init__(
+        self,
+        *,
+        user_id: str,
+        project_id: str,
+        document_id: str,
+        bucket: str,
+        mapping_method: str = "page_indexing",
+        context_format: ContextFormat = "json",
+        access_key_id: str = "",
+        secret_access_key: str = "",
+        endpoint_url: str | None = None,
+        region: str | None = None,
+        client: Any | None = None,
+    ) -> None:
+        if not bucket:
+            raise ValueError("MvlS3Storage requires bucket=")
+        self.user_id = user_id
+        self.project_id = project_id
+        self.document_id = document_id
+        self.bucket = bucket
+        self.mapping_method = mapping_method
+        self.context_format = context_format
+        self.run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        self.progress_log: list[ProgressRecord] = []
+        self._meta: dict[str, Any] = {
+            "user_id": user_id,
+            "project_id": project_id,
+            "document_id": document_id,
+            "run_id": self.run_id,
+            "status": "running",
+        }
+        self._client = client or self._make_client(
+            access_key_id=access_key_id,
+            secret_access_key=secret_access_key,
+            endpoint_url=endpoint_url,
+            region=region,
+        )
+
+    @staticmethod
+    def _make_client(
+        *,
+        access_key_id: str,
+        secret_access_key: str,
+        endpoint_url: str | None,
+        region: str | None,
+    ):
+        try:
+            import boto3
+        except ImportError as exc:
+            raise ImportError(
+                "MvlS3Storage requires boto3. Install with: pip install 'chunksmith-adapters[s3]'"
+            ) from exc
+        kwargs: dict[str, Any] = {}
+        if access_key_id and secret_access_key:
+            kwargs["aws_access_key_id"] = access_key_id
+            kwargs["aws_secret_access_key"] = secret_access_key
+        if endpoint_url:
+            kwargs["endpoint_url"] = endpoint_url
+        if region:
+            kwargs["region_name"] = region
+        return boto3.client("s3", **kwargs)
+
+    def on_progress(self, event: str, payload: dict[str, Any]) -> None:
+        self.progress_log.append(ProgressRecord(event=event, payload=dict(payload)))
+        self._meta["last_event"] = event
+        self._meta["updated_at"] = datetime.now(timezone.utc).isoformat()
+        key = progress_json_object_key(self.user_id, self.project_id, self.document_id, self.run_id)
+        self._put(
+            key, json.dumps([r.__dict__ for r in self.progress_log], ensure_ascii=False, indent=2), "application/json"
+        )
+
+    def _put(self, key: str, body: str | bytes, content_type: str) -> None:
+        data = body.encode("utf-8") if isinstance(body, str) else body
+        self._client.put_object(Bucket=self.bucket, Key=key, Body=data, ContentType=content_type)
+
+    def save_elements(self, elements: list[dict[str, Any]]) -> None:
+        key = elements_json_object_key(self.user_id, self.project_id, self.document_id)
+        self._put(key, json.dumps(elements, ensure_ascii=False, indent=2), "application/json")
+        self._meta["element_count"] = len(elements)
+
+    def save_coded_formate(self, text: str) -> None:
+        key = text_coded_formate_object_key(self.user_id, self.project_id, self.document_id)
+        self._put(key, text, "text/plain")
+
+    def save_outline(self, outline: dict[str, Any]) -> None:
+        key = json_index_object_key(self.user_id, self.project_id, self.document_id)
+        self._put(key, json.dumps(outline, ensure_ascii=False, indent=2), "application/json")
+
+    def save_mapper(self, mapper: dict[str, Any]) -> None:
+        key = mapper_json_object_key(
+            self.document_id,
+            self.mapping_method,
+            user_id=self.user_id,
+            project_id=self.project_id,
+            context_format=self.context_format,
+        )
+        self._put(key, json.dumps(mapper, ensure_ascii=False, indent=2), "application/json")
+
+    def finalize(self) -> None:
+        self._meta["status"] = "completed"
+        self._meta["updated_at"] = datetime.now(timezone.utc).isoformat()
+        self._meta["bucket"] = self.bucket
+        meta_key = f"{self.user_id}/{self.project_id}/json/{self.document_id}_{self.run_id}_meta.json"
+        self._put(meta_key, json.dumps(self._meta, ensure_ascii=False, indent=2), "application/json")

chunksmith_adapters-0.3.0/src/chunksmith_adapters/s3.py

@@ -0,0 +1,50 @@
+"""AWS S3 progressive storage (optional boto3)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from chunksmith_adapters.base import BasePipelineStorage
+
+
+class S3Storage(BasePipelineStorage):
+    """
+    Upload artifacts to S3 as each pipeline stage completes.
+
+    Requires: ``pip install 'chunksmith-adapters[s3]'`` or ``pip install boto3``
+    """
+
+    def __init__(
+        self,
+        *,
+        bucket: str,
+        prefix: str = "chunksmith/",
+        document_id: str,
+        region: str | None = None,
+        client: Any | None = None,
+    ) -> None:
+        if not bucket:
+            raise ValueError("S3Storage requires bucket=")
+        super().__init__(document_id=document_id)
+        self.bucket = bucket
+        self.prefix = prefix.rstrip("/") + "/" if prefix else ""
+        self._client = client or self._make_client(region)
+
+    @staticmethod
+    def _make_client(region: str | None):
+        try:
+            import boto3
+        except ImportError as exc:
+            raise ImportError("S3Storage requires boto3. Install with: pip install 'chunksmith-adapters[s3]'") from exc
+        return boto3.client("s3", region_name=region) if region else boto3.client("s3")
+
+    def _object_key(self, name: str) -> str:
+        return f"{self.prefix}{self._key(name)}"
+
+    def _write_blob(self, name: str, data: bytes, content_type: str) -> None:
+        self._client.put_object(
+            Bucket=self.bucket,
+            Key=self._object_key(name),
+            Body=data,
+            ContentType=content_type,
+        )

chunksmith_adapters-0.3.0/src/chunksmith_adapters.egg-info/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.4
+Name: chunksmith-adapters
+Version: 0.3.0
+Summary: Optional ChunkSmith storage adapters (S3, MongoDB, PostgreSQL, MVL composite).
+Author-email: AnshulParate2004 <anshulnparate@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/AnshulParate2004/ChunkSmith
+Project-URL: Repository, https://github.com/AnshulParate2004/ChunkSmith
+Project-URL: Changelog, https://github.com/AnshulParate2004/ChunkSmith/blob/main/CHANGELOG.md
+Requires-Python: >=3.10
+Requires-Dist: chunksmith-core>=0.3.0
+Provides-Extra: s3
+Requires-Dist: boto3>=1.35.0; extra == "s3"
+Provides-Extra: mongo
+Requires-Dist: pymongo>=4.10.0; extra == "mongo"
+Provides-Extra: postgres
+Requires-Dist: psycopg[binary]>=3.2.0; extra == "postgres"
+Provides-Extra: mvl
+Requires-Dist: boto3>=1.35.0; extra == "mvl"
+Requires-Dist: pymongo>=4.10.0; extra == "mvl"
+Requires-Dist: psycopg[binary]>=3.2.0; extra == "mvl"

chunksmith_adapters-0.3.0/src/chunksmith_adapters.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+pyproject.toml
+src/chunksmith_adapters/__init__.py
+src/chunksmith_adapters/base.py
+src/chunksmith_adapters/config.py
+src/chunksmith_adapters/filesystem.py
+src/chunksmith_adapters/mongodb.py
+src/chunksmith_adapters/mvl_composite.py
+src/chunksmith_adapters/mvl_mongo.py
+src/chunksmith_adapters/mvl_paths.py
+src/chunksmith_adapters/mvl_postgres.py
+src/chunksmith_adapters/mvl_progress.py
+src/chunksmith_adapters/mvl_s3.py
+src/chunksmith_adapters/s3.py
+src/chunksmith_adapters.egg-info/PKG-INFO
+src/chunksmith_adapters.egg-info/SOURCES.txt
+src/chunksmith_adapters.egg-info/dependency_links.txt
+src/chunksmith_adapters.egg-info/requires.txt
+src/chunksmith_adapters.egg-info/top_level.txt

chunksmith_adapters-0.3.0/src/chunksmith_adapters.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

chunksmith_adapters-0.3.0/src/chunksmith_adapters.egg-info/requires.txt

@@ -0,0 +1,15 @@
+chunksmith-core>=0.3.0
+
+[mongo]
+pymongo>=4.10.0
+
+[mvl]
+boto3>=1.35.0
+pymongo>=4.10.0
+psycopg[binary]>=3.2.0
+
+[postgres]
+psycopg[binary]>=3.2.0
+
+[s3]
+boto3>=1.35.0

chunksmith_adapters-0.3.0/src/chunksmith_adapters.egg-info/top_level.txt

@@ -0,0 +1 @@
+chunksmith_adapters