afs-connector-sdk 0.1.0__tar.gz

afs_connector_sdk-0.1.0/.gitignore

@@ -0,0 +1,41 @@
+# --- OS / editor ---
+.DS_Store
+Thumbs.db
+*.swp
+.idea/
+.vscode/
+
+# --- Python (packages land in M0+) ---
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+.uv/
+*.egg-info/
+.ruff_cache/
+.pytest_cache/
+.mypy_cache/
+.ty_cache/
+dist/
+build/
+
+# --- Node (workers/mcp-edge lands later) ---
+node_modules/
+npm-debug.log*
+
+# --- Secrets / local env ---
+.env
+.env.*
+!.env.example
+*.secret.*
+
+# --- Terraform ---
+# Detailed Terraform ignores live in terraform/.gitignore; these catch any
+# stray state/plan artifacts produced outside that tree.
+*.tfstate
+*.tfstate.*
+*.tfplan
+.terraform/
+
+# Agent worktrees (isolated background-agent checkouts)
+.claude/worktrees/

afs_connector_sdk-0.1.0/PKG-INFO

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: afs-connector-sdk
+Version: 0.1.0
+Summary: agentic-fs connector SDK: crawl a source and ingest into agentic-fs. Ships the fs-crawler CLI plus Local FS and S3 connectors.
+Project-URL: Homepage, https://github.com/vivekkhimani/agentic-fs
+Project-URL: Repository, https://github.com/vivekkhimani/agentic-fs
+Project-URL: Issues, https://github.com/vivekkhimani/agentic-fs/issues
+Author-email: Vivek Khimani <vivekkhimani07@gmail.com>
+License-Expression: Apache-2.0
+Keywords: agentic-fs,agents,connector,etl,ingestion,s3
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: afs-core
+Requires-Dist: httpx>=0.27
+Provides-Extra: aws
+Requires-Dist: boto3>=1.34; extra == 'aws'
+Description-Content-Type: text/markdown
+
+# afs-connector-sdk
+
+Crawl a source and ingest its documents into [agentic-fs](https://github.com/vivekkhimani/agentic-fs).
+Ships the `fs-crawler` CLI plus **Local FS** and **S3** connectors.
+
+```bash
+pip install afs-connector-sdk          # Local FS connector + unauthenticated/bearer APIs
+pip install "afs-connector-sdk[aws]"   # adds the S3 connector + SigV4 signing for AWS_IAM Function URLs
+```
+
+## CLI
+
+```bash
+# Crawl a local folder into a dev server
+fs-crawler --connector local --source ./docs \
+  --api-url http://localhost:8080 --namespace docs
+
+# Crawl an S3 prefix into the deployed (AWS_IAM) Function URL
+fs-crawler --connector s3 --source s3://my-bucket/reports/ \
+  --api-url "$FUNCTION_URL" --namespace reports --auth sigv4 --region us-east-1
+
+# Mirror exactly (also delete docs no longer at the source)
+fs-crawler --connector local --source ./docs --api-url "$URL" --namespace docs --prune
+```
+
+Re-runs are cheap and idempotent: a document is skipped unless its content
+checksum differs from what agentic-fs already has, so nothing is re-extracted
+needlessly.
+
+## Library
+
+```python
+from afs_connector_sdk import IngestClient, SyncEngine, SigV4Signer, build_connector
+
+connector = build_connector("local", "./docs")
+async with IngestClient(api_url, signer=SigV4Signer(region="us-east-1")) as client:
+    report = await SyncEngine(client).sync(connector, namespace="docs")
+```
+
+## Writing a connector
+
+Implement `afs_core.contracts.Connector` (`discover()` → `SourceItem`s, `fetch(item)` →
+bytes), certify it against `afs_core.testing.ConnectorConformance`, and register an
+`afs.connectors` entry point. Source-side auth lives in your connector; the SDK
+handles everything else. See [the connector swap guide](../../docs/swap-guides/connectors.md).

afs_connector_sdk-0.1.0/README.md

@@ -0,0 +1,45 @@
+# afs-connector-sdk
+
+Crawl a source and ingest its documents into [agentic-fs](https://github.com/vivekkhimani/agentic-fs).
+Ships the `fs-crawler` CLI plus **Local FS** and **S3** connectors.
+
+```bash
+pip install afs-connector-sdk          # Local FS connector + unauthenticated/bearer APIs
+pip install "afs-connector-sdk[aws]"   # adds the S3 connector + SigV4 signing for AWS_IAM Function URLs
+```
+
+## CLI
+
+```bash
+# Crawl a local folder into a dev server
+fs-crawler --connector local --source ./docs \
+  --api-url http://localhost:8080 --namespace docs
+
+# Crawl an S3 prefix into the deployed (AWS_IAM) Function URL
+fs-crawler --connector s3 --source s3://my-bucket/reports/ \
+  --api-url "$FUNCTION_URL" --namespace reports --auth sigv4 --region us-east-1
+
+# Mirror exactly (also delete docs no longer at the source)
+fs-crawler --connector local --source ./docs --api-url "$URL" --namespace docs --prune
+```
+
+Re-runs are cheap and idempotent: a document is skipped unless its content
+checksum differs from what agentic-fs already has, so nothing is re-extracted
+needlessly.
+
+## Library
+
+```python
+from afs_connector_sdk import IngestClient, SyncEngine, SigV4Signer, build_connector
+
+connector = build_connector("local", "./docs")
+async with IngestClient(api_url, signer=SigV4Signer(region="us-east-1")) as client:
+    report = await SyncEngine(client).sync(connector, namespace="docs")
+```
+
+## Writing a connector
+
+Implement `afs_core.contracts.Connector` (`discover()` → `SourceItem`s, `fetch(item)` →
+bytes), certify it against `afs_core.testing.ConnectorConformance`, and register an
+`afs.connectors` entry point. Source-side auth lives in your connector; the SDK
+handles everything else. See [the connector swap guide](../../docs/swap-guides/connectors.md).

afs_connector_sdk-0.1.0/pyproject.toml

@@ -0,0 +1,49 @@
+[project]
+name = "afs-connector-sdk"
+version = "0.1.0"
+description = "agentic-fs connector SDK: crawl a source and ingest into agentic-fs. Ships the fs-crawler CLI plus Local FS and S3 connectors."
+readme = "README.md"
+requires-python = ">=3.12"
+license = "Apache-2.0"
+authors = [{ name = "Vivek Khimani", email = "vivekkhimani07@gmail.com" }]
+keywords = [
+    "agentic-fs",
+    "agents",
+    "connector",
+    "ingestion",
+    "s3",
+    "etl",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.12",
+    "Typing :: Typed",
+]
+dependencies = [
+    "afs-core",
+    "httpx>=0.27",
+]
+
+[project.optional-dependencies]
+# The S3 connector (boto3) and SigV4 signing for AWS_IAM Function URLs. The Local
+# FS connector and unauthenticated/bearer endpoints need none of this.
+aws = ["boto3>=1.34"]
+
+[project.scripts]
+fs-crawler = "afs_connector_sdk.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/vivekkhimani/agentic-fs"
+Repository = "https://github.com/vivekkhimani/agentic-fs"
+Issues = "https://github.com/vivekkhimani/agentic-fs/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/afs_connector_sdk"]
+
+[tool.uv.sources]
+afs-core = { workspace = true }

afs_connector_sdk-0.1.0/src/afs_connector_sdk/__init__.py

@@ -0,0 +1,22 @@
+"""agentic-fs connector SDK — crawl a source and ingest into agentic-fs.
+
+Public surface: the ``IngestClient`` (HTTP), the ``SyncEngine`` (discover → skip
+unchanged → ingest → prune), request signers, and ``build_connector``. The
+``fs-crawler`` CLI wires them together. Source-specific logic is a
+`afs_core.contracts.Connector`; this package ships Local FS and S3.
+"""
+
+from afs_connector_sdk.auth import NoAuth, RequestSigner, SigV4Signer
+from afs_connector_sdk.client import IngestClient
+from afs_connector_sdk.engine import SyncEngine, SyncReport
+from afs_connector_sdk.registry import build_connector
+
+__all__ = [
+    "IngestClient",
+    "NoAuth",
+    "RequestSigner",
+    "SigV4Signer",
+    "SyncEngine",
+    "SyncReport",
+    "build_connector",
+]

afs_connector_sdk-0.1.0/src/afs_connector_sdk/auth.py

@@ -0,0 +1,61 @@
+"""How the connector authenticates **to the agentic-fs API** (not to the source).
+
+A signer returns the extra headers a request needs. Source-side auth (reaching
+S3 / Google Drive / SharePoint) lives inside each connector, not here. Two
+signers ship today:
+
+- ``NoAuth`` — local dev or an unauthenticated endpoint.
+- ``SigV4Signer`` — the default AWS deployment, whose Function URL uses
+  ``AWS_IAM`` auth. Needs the ``[aws]`` extra. Bearer-token (OAuth) auth arrives
+  with the resource server.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class RequestSigner(Protocol):
+    def headers_for(self, *, method: str, url: str, body: bytes) -> dict[str, str]:
+        """Headers to add so the API accepts the request (may be empty)."""
+        ...
+
+
+class NoAuth:
+    def headers_for(self, *, method: str, url: str, body: bytes) -> dict[str, str]:
+        return {}
+
+
+class SigV4Signer:
+    """Signs requests with AWS SigV4 for an ``AWS_IAM`` Lambda Function URL.
+
+    Credentials come from the standard AWS chain (env, profile, role) unless
+    passed explicitly. The signer is built once and reused across requests.
+    """
+
+    def __init__(self, *, region: str, service: str = "lambda", credentials: Any = None) -> None:
+        try:
+            from botocore.session import Session
+        except ModuleNotFoundError as err:  # pragma: no cover - import guard
+            raise RuntimeError(
+                "SigV4 signing needs the optional extra: pip install 'afs-connector-sdk[aws]'"
+            ) from err
+        self._region = region
+        self._service = service
+        if credentials is None:
+            credentials = Session().get_credentials()
+        if credentials is None:
+            raise RuntimeError(
+                "no AWS credentials found — configure the standard AWS credential chain "
+                "(env vars, a shared profile, or an instance/role)"
+            )
+        self._credentials = credentials
+
+    def headers_for(self, *, method: str, url: str, body: bytes) -> dict[str, str]:
+        from botocore.auth import SigV4Auth
+        from botocore.awsrequest import AWSRequest
+
+        request = AWSRequest(method=method, url=url, data=body or b"")
+        SigV4Auth(self._credentials, self._service, self._region).add_auth(request)
+        return dict(request.headers)

afs_connector_sdk-0.1.0/src/afs_connector_sdk/cli.py

@@ -0,0 +1,90 @@
+"""``fs-crawler`` — crawl a source and ingest its documents into agentic-fs."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from collections.abc import Sequence
+
+from afs_connector_sdk.client import IngestClient
+from afs_connector_sdk.engine import SyncEngine, SyncReport
+from afs_connector_sdk.registry import build_connector
+
+
+def _parse_options(pairs: list[str]) -> dict[str, str]:
+    options: dict[str, str] = {}
+    for pair in pairs:
+        if "=" not in pair:
+            raise ValueError(f"--opt expects KEY=VALUE, got {pair!r}")
+        key, value = pair.split("=", 1)
+        options[key] = value
+    return options
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="fs-crawler",
+        description="Crawl a source and ingest its documents into agentic-fs.",
+    )
+    add = parser.add_argument
+    add("--connector", default="local", help="connector name (local, s3, or a plugin)")
+    add("--source", required=True, help="connector source (a directory, or s3://bucket/prefix)")
+    add("--api-url", required=True, help="agentic-fs API base URL")
+    add("--namespace", required=True, help="target namespace")
+    add("--auth", choices=["none", "sigv4"], default="none", help="how to authenticate to the API")
+    add("--region", default="us-east-1", help="AWS region (for --auth sigv4)")
+    add("--concurrency", type=int, default=8, help="max documents in flight")
+    add("--prune", action="store_true", help="delete agentic-fs docs no longer at the source")
+    add("--dry-run", action="store_true", help="report what would change without writing")
+    parser.add_argument(
+        "--opt",
+        action="append",
+        default=[],
+        metavar="KEY=VALUE",
+        help="connector-specific option (repeatable)",
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+
+    try:
+        connector = build_connector(args.connector, args.source, **_parse_options(args.opt))
+    except (ValueError, RuntimeError) as err:
+        print(f"error: {err}", file=sys.stderr)
+        return 2
+
+    signer = None
+    if args.auth == "sigv4":
+        from afs_connector_sdk.auth import SigV4Signer
+
+        try:
+            signer = SigV4Signer(region=args.region)
+        except RuntimeError as err:
+            print(f"error: {err}", file=sys.stderr)
+            return 2
+
+    report = asyncio.run(_run(connector, args, signer))
+    tag = " (dry-run)" if args.dry_run else ""
+    print(
+        f"{args.connector} -> {args.namespace}{tag}: "
+        f"ingested={report.ingested} skipped={report.skipped} "
+        f"deleted={report.deleted} failed={report.failed}"
+    )
+    for line in report.errors[:20]:
+        print(f"  ! {line}", file=sys.stderr)
+    return 1 if report.failed else 0
+
+
+async def _run(connector: object, args: argparse.Namespace, signer: object) -> SyncReport:
+    async with IngestClient(args.api_url, signer=signer) as client:  # type: ignore[arg-type]
+        engine = SyncEngine(
+            client, concurrency=args.concurrency, prune=args.prune, dry_run=args.dry_run
+        )
+        return await engine.sync(connector, args.namespace)  # type: ignore[arg-type]
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

afs_connector_sdk-0.1.0/src/afs_connector_sdk/client.py

@@ -0,0 +1,99 @@
+"""Async HTTP client for the agentic-fs ingest + read API.
+
+Signing is done by building the httpx request first, then signing its *final*
+URL and attaching the result — so the bytes signed are exactly the bytes sent.
+That avoids the SigV4 query-encoding mismatch you hit when the signed URL and
+the transmitted URL disagree on how a path like ``a/b.md`` is escaped.
+"""
+
+from __future__ import annotations
+
+from types import TracebackType
+from typing import Any
+
+import httpx
+
+from afs_connector_sdk.auth import NoAuth, RequestSigner
+
+
+class IngestClient:
+    def __init__(
+        self, base_url: str, *, signer: RequestSigner | None = None, timeout: float = 30.0
+    ) -> None:
+        self._base = base_url.rstrip("/")
+        self._signer = signer or NoAuth()
+        self._http = httpx.AsyncClient(timeout=timeout)
+
+    async def __aenter__(self) -> IngestClient:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        await self._http.aclose()
+
+    async def _send(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: dict[str, Any] | None = None,
+        content: bytes | None = None,
+        content_type: str | None = None,
+    ) -> httpx.Response:
+        headers = {"content-type": content_type} if content_type else {}
+        request = self._http.build_request(
+            method, f"{self._base}{path}", params=params, content=content, headers=headers
+        )
+        request.headers.update(
+            self._signer.headers_for(
+                method=request.method, url=str(request.url), body=content or b""
+            )
+        )
+        return await self._http.send(request)
+
+    async def put_document(
+        self, namespace: str, path: str, data: bytes, *, content_type: str | None = None
+    ) -> dict[str, Any]:
+        resp = await self._send(
+            "PUT",
+            f"/v1/ingest/{namespace}/doc",
+            params={"path": path},
+            content=data,
+            content_type=content_type,
+        )
+        resp.raise_for_status()
+        return resp.json()
+
+    async def stat(self, namespace: str, path: str) -> dict[str, Any] | None:
+        resp = await self._send("GET", f"/v1/fs/{namespace}/stat", params={"path": path})
+        if resp.status_code == httpx.codes.NOT_FOUND:
+            return None
+        resp.raise_for_status()
+        return resp.json()
+
+    async def delete_document(self, namespace: str, path: str) -> None:
+        resp = await self._send("DELETE", f"/v1/ingest/{namespace}/doc", params={"path": path})
+        if resp.status_code not in (200, 202, 204):
+            resp.raise_for_status()
+
+    async def list_paths(self, namespace: str, prefix: str = "") -> list[str]:
+        paths: list[str] = []
+        cursor: str | None = None
+        while True:
+            params: dict[str, Any] = {"prefix": prefix, "limit": 200}
+            if cursor:
+                params["cursor"] = cursor
+            resp = await self._send("GET", f"/v1/fs/{namespace}/entries", params=params)
+            resp.raise_for_status()
+            page = resp.json()
+            paths.extend(entry["path"] for entry in page.get("items", []))
+            cursor = page.get("next_cursor")
+            if not cursor:
+                return paths

afs_connector_sdk-0.1.0/src/afs_connector_sdk/connectors/__init__.py

@@ -0,0 +1 @@
+"""Builtin connectors. Each is a small `afs_core.contracts.Connector` impl."""

afs_connector_sdk-0.1.0/src/afs_connector_sdk/connectors/local.py

@@ -0,0 +1,46 @@
+"""Local filesystem connector — crawl a directory tree.
+
+Zero dependencies; the reference connector and the easiest way to dogfood the
+whole ingest path (point it at a folder of docs and read them back over MCP).
+"""
+
+from __future__ import annotations
+
+import mimetypes
+from collections.abc import Iterator
+from pathlib import Path
+
+from afs_core.models import SourceItem
+
+
+class LocalConnector:
+    name = "local"
+
+    def __init__(self, source: str, *, follow_symlinks: str = "false") -> None:
+        self._root = Path(source).expanduser().resolve()
+        if not self._root.is_dir():
+            raise ValueError(f"source is not a directory: {self._root}")
+        # Options arrive as strings from the CLI (`--opt follow_symlinks=true`).
+        self._follow = str(follow_symlinks).lower() in {"1", "true", "yes"}
+
+    def discover(self) -> Iterator[SourceItem]:
+        for path in sorted(self._root.rglob("*")):
+            rel = path.relative_to(self._root)
+            # Skip hidden files and dot-directories (.git, .DS_Store, …).
+            if any(part.startswith(".") for part in rel.parts):
+                continue
+            if path.is_symlink() and not self._follow:
+                continue
+            if not path.is_file():
+                continue
+            stat = path.stat()
+            yield SourceItem(
+                path=rel.as_posix(),
+                locator=str(path),
+                size=stat.st_size,
+                content_type=mimetypes.guess_type(path.name)[0],
+                version=f"mtime:{int(stat.st_mtime)}:{stat.st_size}",
+            )
+
+    def fetch(self, item: SourceItem) -> bytes:
+        return Path(item.locator).read_bytes()

afs_connector_sdk-0.1.0/src/afs_connector_sdk/connectors/s3.py

@@ -0,0 +1,59 @@
+"""S3 connector — crawl an ``s3://bucket/prefix`` of documents.
+
+Source-side auth is the standard boto3 chain (env / profile / role), so reading
+from S3 needs no special handling here — that's the connector pattern: each
+source owns its own auth. Needs the ``[aws]`` extra (boto3).
+
+Pass a prefix ending in ``/`` for folder semantics (``s3://bucket/docs/``); the
+prefix is stripped from each key to form the agentic-fs path.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+from urllib.parse import urlparse
+
+from afs_core.models import SourceItem
+
+
+class S3Connector:
+    name = "s3"
+
+    def __init__(
+        self, source: str, *, endpoint_url: str | None = None, region: str | None = None
+    ) -> None:
+        parsed = urlparse(source)
+        if parsed.scheme != "s3" or not parsed.netloc:
+            raise ValueError(f"source must be s3://bucket/prefix, got {source!r}")
+        self._bucket = parsed.netloc
+        self._prefix = parsed.path.lstrip("/")
+        try:
+            import boto3
+        except ModuleNotFoundError as err:  # pragma: no cover - import guard
+            raise RuntimeError(
+                "the s3 connector needs the optional extra: pip install 'afs-connector-sdk[aws]'"
+            ) from err
+        self._s3: Any = boto3.client("s3", endpoint_url=endpoint_url, region_name=region)
+
+    def discover(self) -> Iterator[SourceItem]:
+        paginator = self._s3.get_paginator("list_objects_v2")
+        for page in paginator.paginate(Bucket=self._bucket, Prefix=self._prefix):
+            for obj in page.get("Contents", []):
+                key = obj["Key"]
+                if key.endswith("/"):
+                    continue  # skip "folder" placeholder objects
+                rel = key[len(self._prefix) :] if self._prefix else key
+                rel = rel.lstrip("/")
+                if not rel:
+                    continue
+                yield SourceItem(
+                    path=rel,
+                    locator=key,
+                    size=obj.get("Size"),
+                    version=(obj.get("ETag") or "").strip('"') or None,
+                )
+
+    def fetch(self, item: SourceItem) -> bytes:
+        resp = self._s3.get_object(Bucket=self._bucket, Key=item.locator)
+        return resp["Body"].read()

afs_connector_sdk-0.1.0/src/afs_connector_sdk/engine.py

@@ -0,0 +1,92 @@
+"""The source-agnostic sync engine.
+
+For each item a connector discovers: fetch the bytes, and skip the (expensive)
+ingest if the document is already present unchanged — decided by comparing the
+content checksum against the catalog's, so re-runs are cheap and idempotent and
+nothing is re-extracted needlessly. With ``prune``, documents that have vanished
+from the source are deleted from agentic-fs (the source stays the source of
+truth).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from afs_connector_sdk.client import IngestClient
+    from afs_core.contracts import Connector
+    from afs_core.models import SourceItem
+
+
+@dataclass
+class SyncReport:
+    ingested: int = 0
+    skipped: int = 0
+    deleted: int = 0
+    failed: int = 0
+    errors: list[str] = field(default_factory=list)
+
+
+class SyncEngine:
+    def __init__(
+        self,
+        client: IngestClient,
+        *,
+        concurrency: int = 8,
+        prune: bool = False,
+        dry_run: bool = False,
+    ) -> None:
+        self._client = client
+        self._sem = asyncio.Semaphore(concurrency)
+        self._prune = prune
+        self._dry = dry_run
+
+    async def sync(self, connector: Connector, namespace: str) -> SyncReport:
+        report = SyncReport()
+        items = list(connector.discover())
+        await asyncio.gather(*(self._process(connector, namespace, it, report) for it in items))
+        if self._prune:
+            await self._prune_missing(namespace, {it.path for it in items}, report)
+        return report
+
+    async def _process(
+        self, connector: Connector, namespace: str, item: SourceItem, report: SyncReport
+    ) -> None:
+        async with self._sem:
+            try:
+                data = await asyncio.to_thread(connector.fetch, item)
+                checksum = hashlib.sha256(data).hexdigest()
+                existing = await self._client.stat(namespace, item.path)
+                if existing and existing.get("checksum") == checksum:
+                    report.skipped += 1
+                    return
+                if not self._dry:
+                    await self._client.put_document(
+                        namespace, item.path, data, content_type=item.content_type
+                    )
+                report.ingested += 1
+            except Exception as err:
+                report.failed += 1
+                report.errors.append(f"{item.path}: {err}")
+
+    async def _prune_missing(self, namespace: str, seen: set[str], report: SyncReport) -> None:
+        try:
+            existing = await self._client.list_paths(namespace)
+        except Exception as err:
+            report.errors.append(f"prune-list: {err}")
+            return
+        for path in existing:
+            if path in seen:
+                continue
+            if self._dry:
+                report.deleted += 1
+                continue
+            try:
+                await self._client.delete_document(namespace, path)
+                report.deleted += 1
+            except Exception as err:
+                report.failed += 1
+                report.errors.append(f"delete {path}: {err}")

afs_connector_sdk-0.1.0/src/afs_connector_sdk/registry.py

@@ -0,0 +1,40 @@
+"""Connector registry — builtins + the ``afs.connectors`` entry-point group.
+
+Same pattern as the store and normalizer registries: pick a connector by name.
+Third-party connectors (Google Drive, SharePoint, …) register an entry point
+whose value is a callable ``(source, **options) -> Connector``; they need no
+change here.
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import entry_points
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from afs_core.contracts import Connector
+
+_ENTRY_GROUP = "afs.connectors"
+
+
+def _builtins() -> dict[str, object]:
+    # Imported lazily so a connector's optional deps (boto3 for s3) aren't needed
+    # just to load the registry or use a different connector.
+    from afs_connector_sdk.connectors.local import LocalConnector
+    from afs_connector_sdk.connectors.s3 import S3Connector
+
+    return {"local": LocalConnector, "s3": S3Connector}
+
+
+def build_connector(name: str, source: str, **options: str) -> Connector:
+    """Construct a connector by name over ``source`` (with connector-specific options)."""
+    factory = _builtins().get(name)
+    if factory is None:
+        for ep in entry_points(group=_ENTRY_GROUP):
+            if ep.name == name:
+                factory = ep.load()
+                break
+    if factory is None:
+        available = sorted(_builtins()) + [ep.name for ep in entry_points(group=_ENTRY_GROUP)]
+        raise ValueError(f"unknown connector {name!r}; available: {', '.join(available)}")
+    return factory(source, **options)  # type: ignore[operator]

afs_connector_sdk-0.1.0/tests/test_client.py

@@ -0,0 +1,74 @@
+"""IngestClient HTTP behavior + the sign-the-final-URL property (via MockTransport)."""
+
+from __future__ import annotations
+
+import httpx
+
+from afs_connector_sdk.client import IngestClient
+
+
+class _CaptureSigner:
+    def __init__(self) -> None:
+        self.signed_urls: list[str] = []
+
+    def headers_for(self, *, method: str, url: str, body: bytes) -> dict[str, str]:
+        self.signed_urls.append(url)
+        return {"authorization": "SIGNED"}
+
+
+def _client_with(handler, signer=None) -> IngestClient:
+    client = IngestClient("http://api.test", signer=signer)
+    client._http = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    return client
+
+
+async def test_put_signs_the_exact_url_it_sends() -> None:
+    sent: dict[str, str] = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        sent["url"] = str(request.url)
+        sent["auth"] = request.headers.get("authorization", "")
+        sent["ctype"] = request.headers.get("content-type", "")
+        return httpx.Response(201, json={"path": "a/b.md", "checksum": "abc"})
+
+    signer = _CaptureSigner()
+    client = _client_with(handler, signer)
+    entry = await client.put_document("ns", "a/b.md", b"hello", content_type="text/markdown")
+    await client.aclose()
+
+    assert entry["checksum"] == "abc"
+    assert sent["auth"] == "SIGNED"
+    assert sent["ctype"] == "text/markdown"
+    # The signer must see the byte-identical URL the transport sent (no
+    # re-encoding between signing and sending) — the crux of SigV4 over query paths.
+    assert signer.signed_urls == [sent["url"]]
+
+
+async def test_stat_404_is_none() -> None:
+    client = _client_with(lambda req: httpx.Response(404, json={"detail": "nope"}))
+    assert await client.stat("ns", "missing.md") is None
+    await client.aclose()
+
+
+async def test_list_paths_follows_pagination() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.params.get("cursor") == "c1":
+            return httpx.Response(200, json={"items": [{"path": "b.md"}], "next_cursor": None})
+        return httpx.Response(200, json={"items": [{"path": "a.md"}], "next_cursor": "c1"})
+
+    client = _client_with(handler)
+    assert await client.list_paths("ns") == ["a.md", "b.md"]
+    await client.aclose()
+
+
+async def test_no_auth_adds_no_authorization_header() -> None:
+    seen: dict[str, str | None] = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen["auth"] = request.headers.get("authorization")
+        return httpx.Response(202)
+
+    client = _client_with(handler)  # default NoAuth
+    await client.delete_document("ns", "a.md")
+    await client.aclose()
+    assert seen["auth"] is None

afs_connector_sdk-0.1.0/tests/test_engine.py

@@ -0,0 +1,94 @@
+"""The sync engine's decisions: ingest new, skip unchanged, prune, dry-run, errors."""
+
+from __future__ import annotations
+
+import hashlib
+
+from afs_connector_sdk.engine import SyncEngine
+from afs_core.models import SourceItem
+
+
+class _FakeClient:
+    """Stands in for IngestClient. `existing` maps path -> stored checksum."""
+
+    def __init__(self, existing: dict[str, str] | None = None) -> None:
+        self.existing = existing or {}
+        self.put: list[tuple[str, bytes]] = []
+        self.deleted: list[str] = []
+
+    async def stat(self, namespace: str, path: str) -> dict[str, str] | None:
+        checksum = self.existing.get(path)
+        return {"checksum": checksum} if checksum is not None else None
+
+    async def put_document(
+        self, namespace: str, path: str, data: bytes, *, content_type: str | None = None
+    ) -> dict[str, str]:
+        self.put.append((path, data))
+        return {}
+
+    async def list_paths(self, namespace: str, prefix: str = "") -> list[str]:
+        return list(self.existing)
+
+    async def delete_document(self, namespace: str, path: str) -> None:
+        self.deleted.append(path)
+
+
+class _FakeConnector:
+    name = "fake"
+
+    def __init__(self, items: dict[str, bytes], *, broken: set[str] | None = None) -> None:
+        self._items = items
+        self._broken = broken or set()
+
+    def discover(self) -> list[SourceItem]:
+        return [SourceItem(path=p, locator=p) for p in self._items]
+
+    def fetch(self, item: SourceItem) -> bytes:
+        if item.locator in self._broken:
+            raise OSError("unreadable")
+        return self._items[item.locator]
+
+
+def _sha(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+async def test_ingests_new_documents() -> None:
+    client = _FakeClient()
+    connector = _FakeConnector({"a.md": b"alpha", "sub/b.txt": b"beta"})
+    report = await SyncEngine(client).sync(connector, "ns")
+    assert report.ingested == 2 and report.skipped == 0
+    assert {p for p, _ in client.put} == {"a.md", "sub/b.txt"}
+
+
+async def test_skips_unchanged_by_checksum() -> None:
+    client = _FakeClient(existing={"a.md": _sha(b"alpha")})
+    connector = _FakeConnector({"a.md": b"alpha", "b.md": b"new"})
+    report = await SyncEngine(client).sync(connector, "ns")
+    assert report.skipped == 1 and report.ingested == 1
+    assert [p for p, _ in client.put] == ["b.md"]
+
+
+async def test_dry_run_writes_nothing() -> None:
+    client = _FakeClient()
+    connector = _FakeConnector({"a.md": b"alpha"})
+    report = await SyncEngine(client, dry_run=True).sync(connector, "ns")
+    assert report.ingested == 1
+    assert client.put == []
+
+
+async def test_prune_deletes_documents_absent_from_source() -> None:
+    client = _FakeClient(existing={"gone.md": _sha(b"old")})
+    connector = _FakeConnector({"a.md": b"alpha"})
+    report = await SyncEngine(client, prune=True).sync(connector, "ns")
+    assert report.ingested == 1 and report.deleted == 1
+    assert client.deleted == ["gone.md"]
+
+
+async def test_one_bad_document_does_not_abort_the_crawl() -> None:
+    client = _FakeClient()
+    connector = _FakeConnector({"good.md": b"ok", "bad.md": b"x"}, broken={"bad.md"})
+    report = await SyncEngine(client).sync(connector, "ns")
+    assert report.ingested == 1 and report.failed == 1
+    assert [p for p, _ in client.put] == ["good.md"]
+    assert any("bad.md" in e for e in report.errors)

afs_connector_sdk-0.1.0/tests/test_local_connector.py

@@ -0,0 +1,53 @@
+"""Local FS connector — certified against the afs-core kit, plus its specifics."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from afs_connector_sdk.connectors.local import LocalConnector
+from afs_core.testing import ConnectorConformance
+
+
+def _populate(root: Path) -> None:
+    (root / "a.md").write_text("alpha")
+    (root / "sub").mkdir()
+    (root / "sub" / "b.txt").write_text("beta beta")
+
+
+class TestLocalConnector(ConnectorConformance):
+    @pytest.fixture
+    def connector(self, tmp_path: Path) -> LocalConnector:
+        _populate(tmp_path)
+        return LocalConnector(str(tmp_path))
+
+
+def test_discovers_nested_relative_paths(tmp_path: Path) -> None:
+    _populate(tmp_path)
+    paths = {item.path for item in LocalConnector(str(tmp_path)).discover()}
+    assert paths == {"a.md", "sub/b.txt"}
+
+
+def test_skips_hidden_files_and_dot_dirs(tmp_path: Path) -> None:
+    _populate(tmp_path)
+    (tmp_path / ".secret").write_text("nope")
+    (tmp_path / ".git").mkdir()
+    (tmp_path / ".git" / "config").write_text("nope")
+    paths = {item.path for item in LocalConnector(str(tmp_path)).discover()}
+    assert paths == {"a.md", "sub/b.txt"}
+
+
+def test_fetch_roundtrips_bytes(tmp_path: Path) -> None:
+    _populate(tmp_path)
+    connector = LocalConnector(str(tmp_path))
+    item = next(i for i in connector.discover() if i.path == "a.md")
+    assert connector.fetch(item) == b"alpha"
+    assert item.content_type == "text/markdown"
+
+
+def test_rejects_non_directory(tmp_path: Path) -> None:
+    f = tmp_path / "file.txt"
+    f.write_text("x")
+    with pytest.raises(ValueError, match="not a directory"):
+        LocalConnector(str(f))

afs_connector_sdk-0.1.0/tests/test_s3_connector.py

@@ -0,0 +1,42 @@
+"""S3 connector — certified against the afs-core kit using moto."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import boto3
+import pytest
+from moto import mock_aws
+
+from afs_connector_sdk.connectors.s3 import S3Connector
+from afs_core.testing import ConnectorConformance
+
+
+@pytest.fixture
+def s3_source() -> Iterator[str]:
+    with mock_aws():
+        client = boto3.client("s3", region_name="us-east-1")
+        client.create_bucket(Bucket="docs")
+        client.put_object(Bucket="docs", Key="reports/a.md", Body=b"alpha")
+        client.put_object(Bucket="docs", Key="reports/sub/b.txt", Body=b"beta beta")
+        client.put_object(Bucket="docs", Key="reports/", Body=b"")  # folder placeholder
+        yield "s3://docs/reports/"
+
+
+class TestS3Connector(ConnectorConformance):
+    @pytest.fixture
+    def connector(self, s3_source: str) -> S3Connector:
+        return S3Connector(s3_source, region="us-east-1")
+
+
+def test_strips_prefix_and_skips_placeholders(s3_source: str) -> None:
+    connector = S3Connector(s3_source, region="us-east-1")
+    items = {item.path: item for item in connector.discover()}
+    assert set(items) == {"a.md", "sub/b.txt"}
+    assert connector.fetch(items["a.md"]) == b"alpha"
+    assert items["a.md"].version  # ETag carried as the change token
+
+
+def test_rejects_non_s3_source() -> None:
+    with pytest.raises(ValueError, match="s3://bucket/prefix"):
+        S3Connector("/local/path")