aethergraph 0.1.0a1__py3-none-any.whl

aethergraph/services/artifacts/jsonl_index.py

@@ -0,0 +1,123 @@
+# aethergraph/artifacts/index_jsonl.py
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import threading
+from typing import Literal
+
+from aethergraph.contracts.services.artifacts import Artifact
+
+
+class JsonlArtifactIndexSync:
+    """Simple JSONL-based artifact index for small to medium scale use cases.
+    Not suitable for very large scale (millions of artifacts) due to linear scans.
+    """
+
+    def __init__(self, path: str, occurrences_path: str | None = None):
+        self.path = path
+        self.occ_path = occurrences_path or (os.path.splitext(path)[0] + "_occurrences.jsonl")
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        # small in-memory map for quick lookup / dedup of last write
+        self._by_id = {}
+        self._lock = threading.Lock()
+        if os.path.exists(self.path):
+            with open(self.path, encoding="utf-8") as f:
+                for line in f:
+                    if not line.strip():
+                        continue
+                    rec = json.loads(line)
+                    self._by_id[rec["artifact_id"]] = rec
+
+    def upsert(self, a: Artifact) -> None:
+        """Upsert an artifact record."""
+        with self._lock:
+            rec = a.to_dict()
+            self._by_id[a.artifact_id] = rec
+            with open(self.path, "a", encoding="utf-8") as f:
+                f.write(json.dumps(rec) + "\n")
+
+    def list_for_run(self, run_id: str) -> list[Artifact]:
+        """List all artifacts for a given run_id."""
+        return [Artifact(**r) for r in self._by_id.values() if r.get("run_id") == run_id]
+
+    def search(
+        self,
+        *,
+        kind: str | None = None,
+        labels: dict[str, str] | None = None,
+        metric: str | None = None,
+        mode: Literal["max", "min"] | None = None,
+    ) -> list[Artifact]:
+        """Search artifacts by kind, labels (exact match), and metric (min/max)."""
+        rows = list(self._by_id.values())
+        if kind:
+            rows = [r for r in rows if r.get("kind") == kind]
+        if labels:
+            for k, v in labels.items():
+                rows = [r for r in rows if r.get("labels", {}).get(k) == v]
+        if metric and mode:
+            rows = [r for r in rows if metric in r.get("metrics", {})]
+            rows.sort(key=lambda r: r["metrics"][metric], reverse=(mode == "max"))
+        return [Artifact(**r) for r in rows]
+
+    def best(
+        self,
+        *,
+        kind: str,
+        metric: str,
+        mode: Literal["max", "min"],
+        filters: dict[str, str] | None = None,
+    ) -> Artifact | None:
+        """Get the best artifact by metric with optional filters."""
+        rows = self.search(kind=kind, labels=filters, metric=metric, mode=mode)
+        return rows[0] if rows else None
+
+    def pin(self, artifact_id: str, pinned: bool = True) -> None:
+        """Pin or unpin an artifact by artifact_id."""
+        if artifact_id in self._by_id:
+            self._by_id[artifact_id]["pinned"] = bool(pinned)
+            with open(self.path, "a", encoding="utf-8") as f:
+                f.write(json.dumps(self._by_id[artifact_id]) + "\n")
+
+    def record_occurrence(self, a: Artifact, extra_labels: dict | None = None):
+        """
+        Append-only log that this artifact appeared in this run/node at this time.
+        Keeps lineage even if bytes are identical across runs.
+        """
+        row = {
+            "artifact_id": a.artifact_id,
+            "run_id": a.run_id,
+            "graph_id": a.graph_id,
+            "node_id": a.node_id,
+            "tool_name": a.tool_name,
+            "tool_version": a.tool_version,
+            "created_at": a.created_at,
+            "labels": a.labels | (extra_labels or {}),
+        }
+        with open(self.occ_path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(row) + "\n")
+
+
+class JsonlArtifactIndex:  # implements AsyncArtifactIndex
+    def __init__(self, path: str, occurrences_path: str | None = None):
+        self._sync = JsonlArtifactIndexSync(path, occurrences_path)
+
+    async def upsert(self, a: Artifact) -> None:
+        await asyncio.to_thread(self._sync.upsert, a)
+
+    async def list_for_run(self, run_id: str) -> list[Artifact]:
+        return await asyncio.to_thread(self._sync.list_for_run, run_id)
+
+    async def search(self, **kw) -> list[Artifact]:
+        return await asyncio.to_thread(self._sync.search, **kw)
+
+    async def best(self, **kw) -> Artifact | None:
+        return await asyncio.to_thread(self._sync.best, **kw)
+
+    async def pin(self, artifact_id: str, pinned: bool = True) -> None:
+        await asyncio.to_thread(self._sync.pin, artifact_id, pinned)
+
+    async def record_occurrence(self, a: Artifact, extra_labels: dict | None = None):
+        await asyncio.to_thread(self._sync.record_occurrence, a, extra_labels)

aethergraph/services/artifacts/paths.py

@@ -0,0 +1,23 @@
+# at top of the file
+from pathlib import Path
+from urllib.parse import unquote, urlparse
+from urllib.request import url2pathname
+
+
+def _from_uri_or_path(s: str | Path) -> Path:
+    """Turn a file:// URI or plain path into a local Path (Windows-safe)."""
+    if isinstance(s, Path):
+        return s
+    if not isinstance(s, str):
+        raise TypeError(f"Expected str/Path, got {type(s)}")
+    if "://" not in s:
+        return Path(s)
+    u = urlparse(s)
+    if (u.scheme or "").lower() != "file":
+        # Not a local FS location; return a Path of the original to keep type uniform
+        # Callers can decide what to do; or raise if you want to enforce FS-only.
+        return Path(s)
+    # UNC: file://server/share/path  -> \\server\share\path
+    # Local: file:///C:/path         -> C:\path
+    raw = (f"//{u.netloc}{u.path}") if u.netloc else u.path
+    return Path(url2pathname(unquote(raw)))

aethergraph/services/artifacts/sqlite_index.py

@@ -0,0 +1,209 @@
+# aethergraph/artifacts/index_sqlite.py
+from __future__ import annotations
+
+import asyncio
+import json
+import sqlite3
+from typing import Literal
+
+from aethergraph.contracts.services.artifacts import Artifact
+from aethergraph.services.artifacts.jsonl_index import JsonlArtifactIndexSync
+
+
+class SqliteArtifactIndexSync:
+    """SQLite-based artifact index for medium to large scale use cases.
+    Suitable for larger scale (millions of artifacts) with indexing.
+    """
+
+    def __init__(self, db_path: str):
+        self.db_path = db_path
+        self._init()
+
+    def _init(self):
+        con = sqlite3.connect(self.db_path)
+        cur = con.cursor()
+        cur.execute("""
+        CREATE TABLE IF NOT EXISTS artifacts (
+            artifact_id TEXT PRIMARY KEY,
+            uri TEXT NOT NULL,
+            kind TEXT NOT NULL,
+            bytes INTEGER,
+            sha256 TEXT,
+            mime TEXT,
+            run_id TEXT,
+            graph_id TEXT,
+            node_id TEXT,
+            tool_name TEXT,
+            tool_version TEXT,
+            created_at TEXT,
+            labels TEXT,
+            metrics TEXT,
+            params TEXT,
+            preview_uri TEXT,
+            pinned INTEGER DEFAULT 0
+        )""")
+        cur.execute("CREATE INDEX IF NOT EXISTS idx_kind ON artifacts(kind)")
+        cur.execute("CREATE INDEX IF NOT EXISTS idx_run ON artifacts(run_id)")
+        con.commit()
+        con.close()
+
+    def upsert(self, a: Artifact) -> None:
+        con = sqlite3.connect(self.db_path)
+        cur = con.cursor()
+        cur.execute(
+            """
+        INSERT INTO artifacts
+        (artifact_id, uri, kind, bytes, sha256, mime, run_id, graph_id, node_id,
+         tool_name, tool_version, created_at, labels, metrics, params, preview_uri, pinned)
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+        ON CONFLICT(artifact_id) DO UPDATE SET
+            uri=excluded.uri, kind=excluded.kind, bytes=excluded.bytes,
+            sha256=excluded.sha256, mime=excluded.mime, run_id=excluded.run_id,
+            graph_id=excluded.graph_id, node_id=excluded.node_id,
+            tool_name=excluded.tool_name, tool_version=excluded.tool_version,
+            created_at=excluded.created_at, labels=excluded.labels,
+            metrics=excluded.metrics, params=excluded.params,
+            preview_uri=excluded.preview_uri, pinned=excluded.pinned
+        """,
+            (
+                a.artifact_id,
+                a.uri,
+                a.kind,
+                a.bytes,
+                a.sha256,
+                a.mime,
+                a.run_id,
+                a.graph_id,
+                a.node_id,
+                a.tool_name,
+                a.tool_version,
+                a.created_at,
+                json.dumps(a.labels),
+                json.dumps(a.metrics),
+                json.dumps(a.params),
+                a.preview_uri,
+                1 if a.pinned else 0,
+            ),
+        )
+        con.commit()
+        con.close()
+
+    def list_for_run(self, run_id: str) -> list[Artifact]:
+        con = sqlite3.connect(self.db_path)
+        cur = con.cursor()
+        cur.execute("SELECT * FROM artifacts WHERE run_id=? ORDER BY created_at", (run_id,))
+        rows = cur.fetchall()
+        con.close()
+        return [self._row_to_artifact(r) for r in rows]
+
+    def search(
+        self,
+        *,
+        kind: str | None = None,
+        labels: dict[str, str] | None = None,
+        metric: str | None = None,
+        mode: Literal["max", "min"] | None = None,
+    ) -> list[Artifact]:
+        con = sqlite3.connect(self.db_path)
+        cur = con.cursor()
+        q = "SELECT * FROM artifacts WHERE 1=1"
+        args = []
+        if kind:
+            q += " AND kind=?"
+        args.append(kind)
+        # naive label filter: all requested label kv must be contained in labels json
+        if labels:
+            for k, v in labels.items():
+                q += " AND json_extract(labels, ?) = ?"
+                args += (f"$.{k}", v)
+        cur.execute(q, args)
+        rows = [self._row_to_artifact(r) for r in cur.fetchall()]
+        con.close()
+        if metric and mode and rows:
+            rows = [r for r in rows if r.metrics and metric in r.metrics]
+            reverse = mode == "max"
+            rows.sort(key=lambda a: a.metrics.get(metric, float("-inf")), reverse=reverse)
+        return rows
+
+    def best(
+        self,
+        *,
+        kind: str,
+        metric: str,
+        mode: Literal["max", "min"],
+        filters: dict[str, str] | None = None,
+    ) -> Artifact | None:
+        rows = self.search(kind=kind, labels=filters, metric=metric, mode=mode)
+        return rows[0] if rows else None
+
+    def pin(self, artifact_id: str, pinned: bool = True) -> None:
+        con = sqlite3.connect(self.db_path)
+        cur = con.cursor()
+        cur.execute(
+            "UPDATE artifacts SET pinned=? WHERE artifact_id=?", (1 if pinned else 0, artifact_id)
+        )
+        con.commit()
+        con.close()
+
+    def _row_to_artifact(self, r) -> Artifact:
+        (
+            artifact_id,
+            uri,
+            kind,
+            bytes_,
+            sha256,
+            mime,
+            run_id,
+            graph_id,
+            node_id,
+            tool_name,
+            tool_version,
+            created_at,
+            labels,
+            metrics,
+            params,
+            preview_uri,
+            pinned,
+        ) = r
+        return Artifact(
+            artifact_id=artifact_id,
+            uri=uri,
+            kind=kind,
+            bytes=bytes_,
+            sha256=sha256,
+            mime=mime,
+            run_id=run_id,
+            graph_id=graph_id,
+            node_id=node_id,
+            tool_name=tool_name,
+            tool_version=tool_version,
+            created_at=created_at,
+            labels=json.loads(labels or "{}"),
+            metrics=json.loads(metrics or "{}"),
+            params=json.loads(params or "{}"),
+            preview_uri=preview_uri,
+            pinned=bool(pinned),
+        )
+
+
+class SqliteArtifactIndex:  # implements AsyncArtifactIndex
+    def __init__(self, path: str, occurrences_path: str | None = None):
+        self._sync = JsonlArtifactIndexSync(path, occurrences_path)
+
+    async def upsert(self, a: Artifact) -> None:
+        await asyncio.to_thread(self._sync.upsert, a)
+
+    async def list_for_run(self, run_id: str) -> list[Artifact]:
+        return await asyncio.to_thread(self._sync.list_for_run, run_id)
+
+    async def search(self, **kw) -> list[Artifact]:
+        return await asyncio.to_thread(self._sync.search, **kw)
+
+    async def best(self, **kw) -> Artifact | None:
+        return await asyncio.to_thread(self._sync.best, **kw)
+
+    async def pin(self, artifact_id: str, pinned: bool = True) -> None:
+        await asyncio.to_thread(self._sync.pin, artifact_id, pinned)
+
+    async def record_occurrence(self, a: Artifact, extra_labels: dict | None = None):
+        await asyncio.to_thread(self._sync.record_occurrence, a, extra_labels)

aethergraph/services/artifacts/utils.py

@@ -0,0 +1,124 @@
+import asyncio
+from datetime import datetime, timezone
+from fnmatch import fnmatch
+import hashlib
+import json
+import os
+from pathlib import Path
+
+
+def now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+async def to_thread(fn, *a, **k):
+    return await asyncio.to_thread(fn, *a, **k)
+
+
+# ----- helpers ----- NOTE: we have multiple copies of these in different places, consider centralizing -----
+def _now_iso():
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _sha256_file(path: str, chunk=1024 * 1024) -> tuple[str, int]:
+    """Return (sha256 hex, size in bytes) of a file."""
+    h = hashlib.sha256()
+    total = 0
+    with open(path, "rb") as f:
+        while True:
+            b = f.read(chunk)
+            if not b:
+                break
+            h.update(b)
+            total += len(b)
+    return h.hexdigest(), total
+
+
+def _content_addr_path(base_dir: str, sha256: str, ext: str | None) -> str:
+    """Return a content-addressed path under base_dir for a given sha256 and optional extension.
+    Creates subdirectories as needed.
+
+    It works as follows:
+    - Takes the first 4 characters of the sha256 hash to create two levels of subdirectories.
+    - The first two characters form the first subdirectory (sub1).
+    - The next two characters form the second subdirectory (sub2).
+    - The full sha256 hash, optionally followed by the provided file extension, is used as the filename.
+    - Ensures that the target directory exists by creating it if necessary.
+    - Returns the full path to the content-addressed file.
+
+    The final path structure will look like:
+    base_dir/sub1/sub2/sha256[.ext]
+    """
+    sub1, sub2 = sha256[:2], sha256[2:4]
+    fname = sha256 + (ext or "")
+    target_dir = os.path.join(base_dir, sub1, sub2)
+    os.makedirs(target_dir, exist_ok=True)
+    return os.path.join(target_dir, fname)
+
+
+def _walk_dir(root: str, include: list[str] | None, exclude: list[str] | None):
+    """Yield (relpath, abspath) for files under root honoring include/exclude globs."""
+    root_p = Path(root)
+    for p in root_p.rglob("*"):
+        if not p.is_file():
+            continue
+        rel = str(p.relative_to(root_p)).replace("\\", "/")
+        if exclude and any(fnmatch.fnmatch(rel, pat) for pat in exclude):
+            continue
+        if include and not any(fnmatch.fnmatch(rel, pat) for pat in include):
+            continue
+        yield rel, str(p)
+
+
+def _tree_manifest_and_hash(root: str, include: list[str] | None, exclude: list[str] | None):
+    """
+    Build a deterministic manifest of files: [{"path": rel, "sha256": sha, "bytes": n}, ...]
+    The tree hash is sha256 over JSON lines: "<rel> <sha> <bytes>\n" sorted by rel.
+    """
+    entries = []
+    lines = []
+    for rel, abspath in _walk_dir(root, include, exclude):
+        sha, nbytes = _sha256_file(abspath)
+        entries.append({"path": rel, "sha256": sha, "bytes": nbytes})
+        lines.append(f"{rel}\t{sha}\t{nbytes}\n")
+    # sort for determinism
+    lines.sort()
+    h = hashlib.sha256()
+    for line in lines:
+        h.update(line.encode("utf-8"))
+    tree_sha = h.hexdigest()
+    return entries, tree_sha
+
+
+def _content_addr_dir_path(base_dir: str, tree_sha: str):
+    # content-addressed folder to hold manifest (and optional archive)
+    sub1, sub2 = tree_sha[:2], tree_sha[2:4]
+    target_dir = os.path.join(base_dir, sub1, sub2, tree_sha)
+    os.makedirs(target_dir, exist_ok=True)
+    return target_dir
+
+
+def _write_json(path: str, obj: dict | list):
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(obj, f, ensure_ascii=False, separators=(",", ":"))
+
+
+def _maybe_cleanup_tmp_parent(tmp_root: str, path: str):
+    """Remove empty parent dirs strictly under tmp_root (never _tmp itself)."""
+    try:
+        parent = os.path.dirname(os.path.abspath(path))
+        tmp_root_abs = os.path.abspath(tmp_root)
+
+        # Only operate if `parent` is inside tmp_root
+        while (
+            os.path.commonpath([parent, tmp_root_abs]) == tmp_root_abs
+            and os.path.normcase(parent)
+            != os.path.normcase(tmp_root_abs)  # don't delete _tmp itself
+        ):
+            try:
+                os.rmdir(parent)  # only removes if empty
+            except OSError:
+                break
+            parent = os.path.dirname(parent)
+    except Exception:
+        pass

aethergraph/services/auth/dev.py

@@ -0,0 +1,16 @@
+# services/auth/dev.py
+class DevTokenAuthn:
+    """Development token authenticator. Accepts any token, returns 'dev' as subject."""
+
+    def __init__(self, header="x-dev-token"):
+        self.header = header
+
+    async def whoami(self, token: str | None) -> dict:
+        return {"subject": token or "dev", "roles": ["admin"]}
+
+
+class AllowAllAuthz:
+    """Development authorizer that allows all actions."""
+
+    async def allow(self, actor: dict, action: str, resource: str) -> bool:
+        return True