aethergraph 0.1.0a2__py3-none-any.whl → 0.1.0a4__py3-none-any.whl

aethergraph/services/skills/utils.py

@@ -0,0 +1,194 @@
+from pathlib import Path
+import re
+from typing import Any
+
+from .skills import Skill
+
+try:
+    import yaml  # type: ignore
+except ImportError:  # pragma: no cover
+    yaml = None  # TODO: enforce PyYAML as a dependency?
+
+_FRONT_MATTER_DELIM = re.compile(r"^---\s*$")
+# Only treat H2 (##) as section delimiters, per spec.
+_SECTION_HEADING_RE = re.compile(r"^(##)\s+(.*)$")
+
+
+def _split_front_matter(text: str) -> tuple[dict[str, Any], str]:
+    """
+    Split YAML front matter from the rest of the markdown body.
+
+    Expects:
+        ---
+        yaml: here
+        ---
+        # Markdown starts here
+
+    Returns: (meta_dict, body_markdown)
+    """
+    lines = text.splitlines()
+    if not lines or not _FRONT_MATTER_DELIM.match(lines[0].strip()):
+        # No front matter block
+        return {}, text
+
+    # Find closing '---'
+    end_idx = None
+    for i in range(1, len(lines)):
+        if _FRONT_MATTER_DELIM.match(lines[i].strip()):
+            end_idx = i
+            break
+
+    if end_idx is None:
+        # Malformed front matter; treat entire file as body
+        return {}, text
+
+    fm_lines = lines[1:end_idx]
+    body_lines = lines[end_idx + 1 :]
+
+    fm_str = "\n".join(fm_lines)
+    body = "\n".join(body_lines)
+
+    if yaml is None:
+        # If PyYAML is not installed, return empty meta.
+        return {}, body
+
+    try:
+        meta = yaml.safe_load(fm_str) or {}
+    except Exception as exc:
+        # Surface a clear error – this is almost always a YAML indentation / syntax issue
+        raise ValueError(
+            f"Failed to parse YAML front matter: {exc!r}\n" f"Front matter was:\n{fm_str}"
+        ) from exc
+
+    if not isinstance(meta, dict):
+        raise ValueError(
+            f"YAML front matter must be a mapping (dict), got {type(meta)} instead.\n"
+            f"Front matter was:\n{fm_str}"
+        )
+
+    return meta, body
+
+
+def _normalize_section_key(heading: str) -> str:
+    """
+    Normalize a heading text into a section key.
+
+    Rules:
+      - If the heading already contains a dot (e.g. "chat.system"), keep as-is.
+      - Else, lowercase and replace spaces with underscores, e.g. "Chat System" -> "chat_system".
+    """
+    raw = heading.strip()
+    if "." in raw:
+        return raw.strip()
+    return raw.lower().replace(" ", "_")
+
+
+def _split_sections_from_body(body: str) -> dict[str, str]:
+    """
+    Split markdown body into sections keyed by normalized heading.
+
+    - Intro text before any heading -> section "body"
+    - Only H2 headings (`## something`) start new sections.
+    - H3+ (`### ...`) are treated as content.
+    """
+    sections: dict[str, list[str]] = {}
+    current_key: str | None = None
+    buffer: list[str] = []
+    preface: list[str] = []
+
+    lines = body.splitlines()
+
+    for line in lines:
+        m = _SECTION_HEADING_RE.match(line)
+        if m:
+            # Flush previous section or preface
+            if current_key is None:
+                if buffer:
+                    preface.extend(buffer)
+            else:
+                sections[current_key] = sections.get(current_key, []) + buffer
+
+            buffer = []
+            heading = m.group(2).strip()
+            current_key = _normalize_section_key(heading)
+        else:
+            buffer.append(line)
+
+    # Flush last buffer
+    if current_key is None:
+        if buffer:
+            preface.extend(buffer)
+    else:
+        sections[current_key] = sections.get(current_key, []) + buffer
+
+    out: dict[str, str] = {}
+    if preface:
+        out["body"] = "\n".join(preface).strip()
+
+    for k, lines_ in sections.items():
+        text = "\n".join(lines_).strip()
+        if text:
+            out[k] = text
+
+    return out
+
+
+def parse_skill_markdown(text: str, path: Path | None = None) -> Skill:
+    """
+    Parse a single markdown file into a Skill.
+
+    The file must have YAML front matter with at least:
+      - id: string
+      - title: string
+
+    Sections are defined by `## section.key` headings.
+    """
+    meta, body = _split_front_matter(text)
+    sections = _split_sections_from_body(body)
+
+    location = str(path) if path is not None else "<string>"
+
+    # ---- Basic validation of YAML meta ----
+    if not meta:
+        raise ValueError(
+            f"Skill file {location} has no YAML front matter. "
+            "Expected at least:\n"
+            "---\n"
+            "id: some.id\n"
+            "title: Some title\n"
+            "---"
+        )
+
+    skill_id = meta.get("id")
+    title = meta.get("title")
+
+    if not isinstance(skill_id, str) or not skill_id.strip():
+        raise ValueError(
+            f"Skill file {location} is missing a valid 'id' in front matter. "
+            f"Got id={skill_id!r} in:\n{meta}"
+        )
+
+    if not isinstance(title, str) or not title.strip():
+        raise ValueError(
+            f"Skill file {location} is missing a valid 'title' in front matter. "
+            f"Got title={title!r} in:\n{meta}"
+        )
+
+    try:
+        skill = Skill.from_dict(
+            meta=meta,
+            sections=sections,
+            raw_markdown=text,
+            path=path,
+        )
+    except Exception as e:
+        raise ValueError(f"Failed to construct Skill from {location}: {e}") from e
+
+    # Extra guard – if Skill.from_dict ever returns None, fail loudly.
+    if skill is None:
+        raise ValueError(
+            f"Skill.from_dict returned None for skill file {location}. "
+            f"Front matter: {meta}, sections: {list(sections.keys())}"
+        )
+
+    return skill

aethergraph/storage/artifacts/artifact_index_jsonl.py

@@ -75,20 +75,26 @@ class JsonlArtifactIndexSync:
         if labels:
             for k, v in labels.items():
                 if k in TENANT_KEYS:
-                    # Match against top-level JSON fields
-                    rows = [r for r in rows if r.get(k) == v]
-                    continue
-
-                # Normal label filters
-                if isinstance(v, list):
+                    sv = str(v)
                     rows = [
                         r
                         for r in rows
-                        if isinstance(r.get("labels", {}).get(k), list)
-                        and set(v).issubset(set(r["labels"][k]))
+                        if str(r.get(k) or "") == sv
+                        or str((r.get("labels") or {}).get(k) or "") == sv
                     ]
-                else:
-                    rows = [r for r in rows if r.get("labels", {}).get(k) == v]
+                    continue
+
+                # Normal label filters
+                if k == "tags":
+                    want = v if isinstance(v, list) else [v]
+                    want = [str(t).strip() for t in want if str(t).strip()]
+                    if want:
+                        rows = [
+                            r
+                            for r in rows
+                            if any(t in (r.get("labels", {}).get("tags") or []) for t in want)
+                        ]
+                    continue
 
         if metric and mode:
             rows = [r for r in rows if metric in r.get("metrics", {})]

aethergraph/storage/artifacts/artifact_index_sqlite.py

@@ -273,9 +273,19 @@ class SqliteArtifactIndexSync:
                         where.append("(" + " OR ".join(ors) + ")")
                     continue
 
+                # if k in TENANT_KEYS:
+                #     where.append(f"{TENANT_KEYS[k]} = ?")
+                #     params.append(v)
+                #     continue
+
                 if k in TENANT_KEYS:
-                    where.append(f"{TENANT_KEYS[k]} = ?")
-                    params.append(v)
+                    col = TENANT_KEYS[k]
+                    sv = str(v)
+
+                    # column OR labels_json fallback
+                    where.append(f"({col} = ? OR labels_json LIKE ?)")
+                    params.append(sv)
+                    params.append(f'%"{k}": "{sv}"%')
                     continue
 
                 where.append("labels_json LIKE ?")

aethergraph/storage/docstore/sqlite_doc_sync.py

@@ -67,7 +67,7 @@ class SQLiteDocStoreSync:
                     (doc_id, payload, now),
                 )
             except sqlite3.Error as e:
-                print("🍓 SQLiteDocStoreSync ERROR during put:", doc_id, repr(e))
+                print("SQLiteDocStoreSync ERROR during put:", doc_id, repr(e))
                 raise
 
     def get(self, doc_id: str) -> dict[str, Any] | None:

aethergraph/storage/memory/event_persist.py

@@ -47,9 +47,9 @@ class EventLogPersistence(Persistence):
         return f"{self._prefix}{doc_id}"
 
     # --------- API ---------
-    async def append_event(self, run_id: str, evt: Event) -> None:
+    async def append_event(self, scope_id: str, evt: Event) -> None:
         payload = asdict(evt)
-        payload.setdefault("scope_id", run_id)
+        payload.setdefault("scope_id", scope_id)
         payload.setdefault("kind", "memory")
         # you can add tags like ["mem"] if useful
         await self._log.append(payload)
@@ -66,3 +66,43 @@ class EventLogPersistence(Persistence):
         if doc is None:
             raise FileNotFoundError(f"Memory JSON not found for URI: {uri}")
         return doc
+
+    async def get_events_by_ids(
+        self,
+        scope_id: str,
+        event_ids: list[str],
+    ) -> list[Event]:
+        """
+        Fetch events for a given scope_id (timeline) by event_id.
+
+        Implementation v0: use EventLog.query and filter in Python.
+        For moderate timeline sizes and small event_ids lists, this is fine.
+        Later, you can optimize by adding a direct get_many API on EventLog
+        or indexing by (scope_id, event_id).
+        """
+        if not event_ids:
+            return []
+
+        # Fetch all events for the scope_id; TODO: add reasonable limits / paging
+        rows = await self._log.query(
+            scope_id=scope_id,
+            since=None,
+            until=None,
+            kinds=None,
+            tags=None,
+            limit=None,
+            offset=0,
+        )
+
+        by_id: dict[str, Event] = {}
+        for row in rows:
+            eid = row.get("event_id")
+            if eid:
+                by_id[eid] = row
+
+        result: list[Event] = []
+        for eid in event_ids:
+            row = by_id.get(eid)
+            if row is not None:
+                result.append(Event(**row))
+        return result

aethergraph/storage/memory/fs_persist.py

@@ -30,9 +30,9 @@ class FSPersistence(Persistence):
 
     # ---------- Event log (append-only JSONL) ----------
 
-    async def append_event(self, run_id: str, evt: Event) -> None:
+    async def append_event(self, scope_id: str, evt: Event) -> None:
         day = time.strftime("%Y-%m-%d", time.gmtime())
-        path = self.base_dir / "mem" / run_id / "events" / f"{day}.jsonl"
+        path = self.base_dir / "mem" / scope_id / "events" / f"{day}.jsonl"
 
         def _write() -> None:
             path.parent.mkdir(parents=True, exist_ok=True)
@@ -116,3 +116,33 @@ class FSPersistence(Persistence):
                 return json.load(f)
 
         return await asyncio.to_thread(_read)
+
+    async def get_events_by_ids(
+        self,
+        scope_id: str,
+        event_ids: list[str],
+    ) -> list[Event]:
+        """
+        Fetch events for a given scope_id (timeline) by event_id.
+        """
+        id_set = set(event_ids)
+        found: list[Event] = []
+
+        day = time.strftime("%Y-%m-%d", time.gmtime())
+        path = self.base_dir / "mem" / scope_id / "events" / f"{day}.jsonl"
+
+        if not path.exists():
+            return found
+
+        def _read() -> list[Event]:
+            results: list[Event] = []
+            with self._lock, path.open("r", encoding="utf-8") as f:
+                for line in f:
+                    data = json.loads(line)
+                    if data.get("event_id") in id_set:
+                        evt = Event(**data)
+                        results.append(evt)
+            return results
+
+        found = await asyncio.to_thread(_read)
+        return found

aethergraph/storage/search_backend/generic_vector_backend.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from time import time
+from typing import Any
+
+from aethergraph.contracts.services.llm import EmbeddingClientProtocol
+from aethergraph.contracts.storage.search_backend import ScoredItem, SearchBackend
+from aethergraph.contracts.storage.vector_index import PROMOTED_FIELDS, VectorIndex
+
+from .utils import _parse_time_window
+
+
+@dataclass
+class GenericVectorSearchBackend(SearchBackend):
+    """
+    SearchBackend implementation on top of a VectorIndex + EmbeddingClient.
+
+    - Upserts: embed text and store (vector, metadata) in the index
+    - Search: embed query, retrieve top-k by cosine similarity,
+      then apply Python-level metadata filters.
+    """
+
+    index: VectorIndex
+    embedder: EmbeddingClientProtocol
+
+    # -------- helpers ----------------------------------------------------
+    async def _embed(self, text: str) -> list[float]:
+        vec: Sequence[float] = await self.embedder.embed_one(text)
+        # Ensure a concrete list[float] for numpy/etc
+        return [float(x) for x in vec]
+
+    @staticmethod
+    def _match_value(mv: Any, val: Any) -> bool:
+        """
+        Rich matching semantics for filters:
+        - If val is list/tuple/set:
+            - if mv is list-like too -> match if intersection is non-empty
+            - else                    -> match if mv is in val
+        - If val is scalar:
+            - if mv is list-like      -> match if val is in mv
+            - else                    -> match if mv == val
+        """
+        if val is None:
+            return True
+
+        def _is_list_like(x: Any) -> bool:
+            return isinstance(x, (list, tuple, set))  # noqa: UP038
+
+        if _is_list_like(val):
+            if _is_list_like(mv):
+                # any overlap between filter values and meta values
+                return any(x in val for x in mv)
+            else:
+                # meta is scalar, filter is list-like
+                return mv in val
+
+        # val is scalar
+        if _is_list_like(mv):
+            return val in mv
+
+        return mv == val
+
+    @staticmethod
+    def _matches_filters(meta: dict[str, Any], filters: dict[str, Any]) -> bool:
+        """
+        Simple AND filter: all filter keys must match exactly.
+        - If filter value is a list, meta[key] must be in that list.
+        - If filter value is None, we don't constrain that key.
+        """
+        for k, v in filters.items():
+            if v is None:
+                continue
+            if k not in meta:
+                return False
+            mv = meta[k]
+            if not GenericVectorSearchBackend._match_value(mv, v):
+                return False
+        return True
+
+    # -------- public APIs ------------------------------------------------
+    async def upsert(
+        self,
+        *,
+        corpus: str,
+        item_id: str,
+        text: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        if not text:
+            # avoid zero vector; caller should ensure text is non-empty
+            text = ""
+
+        vector = await self._embed(text)
+        await self.index.add(
+            corpus_id=corpus,
+            chunk_ids=[item_id],
+            vectors=[vector],
+            metas=[metadata],
+        )
+
+    async def search(
+        self,
+        *,
+        corpus: str,
+        query: str,
+        top_k: int = 10,
+        filters: dict[str, Any] | None = None,
+        time_window: str | None = None,
+        created_at_min: float | None = None,
+        created_at_max: float | None = None,
+    ) -> list[ScoredItem]:
+        filters = filters or {}
+        if not query.strip():
+            return []
+
+        q_vec = await self._embed(query)
+
+        # ---- 1) Handle time constraints ---------------------------------
+        now_ts = time()
+
+        # If time_window is provided and no explicit min, interpret it as [now - window, now]
+        if time_window and created_at_min is None:
+            duration = _parse_time_window(time_window)
+            created_at_min = now_ts - duration
+
+        # If max is not provided but we used a time_window, default to now
+        if time_window and created_at_max is None:
+            created_at_max = now_ts
+
+        # ---- 2) Split filters into index-level vs Python-level ---------
+        index_filters: dict[str, Any] = {}
+        post_filters: dict[str, Any] = {}
+
+        for key, val in filters.items():
+            if val is None:
+                continue
+
+            if key in PROMOTED_FIELDS and not isinstance(val, (list, tuple, set)):  # noqa: UP038
+                index_filters[key] = val
+            else:
+                post_filters[key] = val
+
+        # ---- 3) Ask index for scoped, time-bounded candidates ----------
+        raw_k = max(top_k * 3, top_k)
+        max_candidates = max(top_k * 50, raw_k)  # tunable safety cap
+
+        rows = await self.index.search(
+            corpus_id=corpus,
+            query_vec=q_vec,
+            k=raw_k,
+            where=index_filters,
+            max_candidates=max_candidates,
+            created_at_min=created_at_min,
+            created_at_max=created_at_max,
+        )
+
+        # ---- 4) Apply Python-level filters + build ScoredItem list -----
+        results: list[ScoredItem] = []
+        for row in rows:
+            chunk_id = row["chunk_id"]
+            score = float(row["score"])
+            meta = dict(row.get("meta") or {})
+
+            if post_filters and not self._matches_filters(meta, post_filters):
+                continue
+
+            results.append(
+                ScoredItem(
+                    item_id=chunk_id,
+                    corpus=corpus,
+                    score=score,
+                    metadata=meta,
+                )
+            )
+            if len(results) >= top_k:
+                break
+
+        return results
+
+    async def search_old(
+        self,
+        *,
+        corpus: str,
+        query: str,
+        top_k: int = 10,
+        filters: dict[str, Any] | None = None,
+    ) -> list[ScoredItem]:
+        """
+        1) Embed the query
+        2) Vector search in the underlying index
+        3) Apply metadata filters in Python
+        """
+        filters = filters or {}
+        if not query.strip():
+            # empty query: probably return nothing for now
+            return []
+
+        q_vec = await self._embed(query)
+
+        # Ask underlying VectorIndex for more than top_k, since we may
+        # filter some out. Factor 3 is arbitrary but usually safe.
+        raw_k = max(top_k * 3, top_k)
+        rows = await self.index.search(
+            corpus_id=corpus,
+            query_vec=q_vec,
+            k=raw_k,
+        )
+
+        results: list[ScoredItem] = []
+        for row in rows:
+            chunk_id = row["chunk_id"]
+            score = float(row["score"])
+            meta = dict(row.get("meta") or {})
+
+            if filters and not self._matches_filters(meta, filters):
+                continue
+
+            results.append(
+                ScoredItem(
+                    item_id=chunk_id,
+                    corpus=corpus,
+                    score=score,
+                    metadata=meta,
+                )
+            )
+            if len(results) >= top_k:
+                break
+        return results

aethergraph/storage/search_backend/null_backend.py

@@ -0,0 +1,34 @@
+from dataclasses import dataclass
+from typing import Any
+
+from aethergraph.contracts.storage.search_backend import ScoredItem, SearchBackend
+
+
+@dataclass
+class NullSearchBackend(SearchBackend):
+    """A no-op search backend that performs no indexing or searching."""
+
+    async def upsert(
+        self,
+        *,
+        corpus: str,
+        item_id: str,
+        text: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        # no-op
+        return
+
+    async def search(
+        self,
+        *,
+        corpus: str,
+        query: str,
+        top_k: int = 10,
+        filters: dict[str, Any] | None = None,
+        time_window: str | None = None,
+        created_at_min: float | None = None,
+        created_at_max: float | None = None,
+    ) -> list[ScoredItem]:
+        # either empty or raise FeatureDisabled
+        return []