crewai-memory-firestore 0.1.0__tar.gz

crewai_memory_firestore-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+.venv/
+dist/
+build/
+uv.lock
+.env

crewai_memory_firestore-0.1.0/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.5
+Name: crewai-memory-firestore
+Version: 0.1.0
+Summary: Google Firestore StorageBackend for CrewAI unified Memory — hierarchical scopes, categories, metadata filters and native vector search (find_nearest)
+Project-URL: Homepage, https://github.com/skamalj/crewai-memory
+Project-URL: Repository, https://github.com/skamalj/crewai-memory.git
+Project-URL: Documentation, https://skamalj.github.io/agentstate-reducer/
+Author-email: Kamal <skamalj@gmail.com>
+Keywords: agent-memory,crewai,firestore,gcp,long-term-memory,memory,storage-backend,vector-search
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: crewai-memory-core>=0.1.0
+Requires-Dist: google-cloud-firestore>=2.16
+Description-Content-Type: text/markdown
+
+# crewai-memory-firestore
+
+A **Google Firestore** `StorageBackend` for [CrewAI](https://docs.crewai.com/en/concepts/memory)'s unified `Memory` — hierarchical scopes, categories, metadata filters, importance/recency, and **native vector search** via `find_nearest`.
+
+```bash
+pip install crewai-memory-firestore
+```
+
+```python
+from crewai import Crew
+from crewai.memory import Memory
+from crewai_memory_firestore import FirestoreMemoryBackend
+
+backend = FirestoreMemoryBackend(project_id="my-project", collection="crewai_memory", dimensions=3072)  # match your embedder
+memory = Memory(storage=backend)
+crew = Crew(agents=[...], tasks=[...], memory=memory)
+```
+
+Or once for every `Crew(memory=True)`: `set_memory_storage_factory(lambda spec: FirestoreMemoryBackend(...))`. Authentication uses Application Default Credentials.
+
+## How it works
+
+- One collection, one document per record (doc id = record id) with the embedding stored as a Firestore `Vector`.
+- `search` uses native `find_nearest` (cosine) after a scope pre-filter. A scope *subtree* is two queries (the exact scope, and the `scope/…` range) merged by similarity; score is `1 - distance`. `categories` / `metadata_filter` / `min_score` are applied on the candidates.
+- Firestore requires a **composite vector index** `scope ASC + embedding (flat, dims)`. With `create_index=True` (default) the backend creates it through the Admin API if missing; the build is asynchronous and vector queries fail with `FAILED_PRECONDITION` until it is READY (a few minutes). Needs `datastore.indexes.create`/`list`, or create it yourself and pass `create_index=False`:
+
+```bash
+gcloud firestore indexes composite create --collection-group=crewai_memory --query-scope=COLLECTION \
+  --field-config=order=ASCENDING,field-path=scope \
+  --field-config=vector-config='{"dimension":"3072","flat":"{}"}',field-path=embedding
+```
+
+Docs: <https://skamalj.github.io/agentstate-reducer/> · part of [crewai-memory](https://github.com/skamalj/crewai-memory)
+
+## License
+
+MIT

crewai_memory_firestore-0.1.0/README.md

@@ -0,0 +1,37 @@
+# crewai-memory-firestore
+
+A **Google Firestore** `StorageBackend` for [CrewAI](https://docs.crewai.com/en/concepts/memory)'s unified `Memory` — hierarchical scopes, categories, metadata filters, importance/recency, and **native vector search** via `find_nearest`.
+
+```bash
+pip install crewai-memory-firestore
+```
+
+```python
+from crewai import Crew
+from crewai.memory import Memory
+from crewai_memory_firestore import FirestoreMemoryBackend
+
+backend = FirestoreMemoryBackend(project_id="my-project", collection="crewai_memory", dimensions=3072)  # match your embedder
+memory = Memory(storage=backend)
+crew = Crew(agents=[...], tasks=[...], memory=memory)
+```
+
+Or once for every `Crew(memory=True)`: `set_memory_storage_factory(lambda spec: FirestoreMemoryBackend(...))`. Authentication uses Application Default Credentials.
+
+## How it works
+
+- One collection, one document per record (doc id = record id) with the embedding stored as a Firestore `Vector`.
+- `search` uses native `find_nearest` (cosine) after a scope pre-filter. A scope *subtree* is two queries (the exact scope, and the `scope/…` range) merged by similarity; score is `1 - distance`. `categories` / `metadata_filter` / `min_score` are applied on the candidates.
+- Firestore requires a **composite vector index** `scope ASC + embedding (flat, dims)`. With `create_index=True` (default) the backend creates it through the Admin API if missing; the build is asynchronous and vector queries fail with `FAILED_PRECONDITION` until it is READY (a few minutes). Needs `datastore.indexes.create`/`list`, or create it yourself and pass `create_index=False`:
+
+```bash
+gcloud firestore indexes composite create --collection-group=crewai_memory --query-scope=COLLECTION \
+  --field-config=order=ASCENDING,field-path=scope \
+  --field-config=vector-config='{"dimension":"3072","flat":"{}"}',field-path=embedding
+```
+
+Docs: <https://skamalj.github.io/agentstate-reducer/> · part of [crewai-memory](https://github.com/skamalj/crewai-memory)
+
+## License
+
+MIT

crewai_memory_firestore-0.1.0/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "crewai-memory-firestore"
+version = "0.1.0"
+description = "Google Firestore StorageBackend for CrewAI unified Memory — hierarchical scopes, categories, metadata filters and native vector search (find_nearest)"
+authors = [{name = "Kamal", email = "skamalj@gmail.com"}]
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "crewai-memory-core>=0.1.0",
+    "google-cloud-firestore>=2.16",
+]
+keywords = ["crewai", "memory", "storage-backend", "firestore", "gcp", "long-term-memory", "agent-memory", "vector-search"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[project.urls]
+Homepage = "https://github.com/skamalj/crewai-memory"
+Repository = "https://github.com/skamalj/crewai-memory.git"
+Documentation = "https://skamalj.github.io/agentstate-reducer/"
+
+[dependency-groups]
+dev = ["pytest>=7.0", "pytest-asyncio"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/crewai_memory_firestore"]

crewai_memory_firestore-0.1.0/src/crewai_memory_firestore/__init__.py

@@ -0,0 +1,166 @@
+"""Google Firestore ``StorageBackend`` for CrewAI unified ``Memory``.
+
+One collection, one document per ``MemoryRecord`` (doc id = record id) with the
+embedding stored as a Firestore ``Vector``. ``search`` uses native
+``find_nearest`` (cosine) after a scope pre-filter; a scope *subtree* needs two
+queries (the exact scope, and the ``scope/…`` range), merged by similarity.
+Built on ``crewai-memory-core``.
+
+Firestore needs a composite **vector index** ``scope ASC + embedding (flat)``;
+with ``create_index=True`` (default) it is created through the Admin API if
+missing (asynchronous build — vector queries fail with FAILED_PRECONDITION
+until READY).
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import List, Optional, Tuple
+
+from google.cloud import firestore
+from google.cloud.firestore_v1.base_query import FieldFilter
+from google.cloud.firestore_v1.base_vector_query import DistanceMeasure
+from google.cloud.firestore_v1.vector import Vector
+
+from crewai_memory_core import MemoryBackend, ROW_FIELDS, norm_scope
+
+__all__ = ["FirestoreMemoryBackend"]
+
+logger = logging.getLogger(__name__)
+_HIGH = chr(0xF8FF)
+
+
+class FirestoreMemoryBackend(MemoryBackend):
+    """CrewAI ``StorageBackend`` on Firestore with native vector search.
+
+    Example:
+        ```python
+        from crewai.memory import Memory
+        from crewai_memory_firestore import FirestoreMemoryBackend
+
+        backend = FirestoreMemoryBackend(project_id="my-project", collection="crewai_memory", dimensions=3072)
+        memory = Memory(storage=backend)
+        ```
+    """
+
+    supports_native_vector_search = True
+
+    def __init__(
+        self,
+        *,
+        project_id: str,
+        collection: str = "crewai_memory",
+        dimensions: int = 3072,
+        client: Optional["firestore.Client"] = None,
+        create_index: bool = True,
+        database: str = "(default)",
+    ) -> None:
+        self.dimensions = dimensions
+        self._project = project_id
+        self._database = database
+        self._collection = collection
+        self._client = client or firestore.Client(project=project_id, database=database)
+        self._col = self._client.collection(collection)
+        if create_index:
+            self._ensure_vector_index()
+
+    # ------------------------------------------------------------------ index bootstrap
+    def _ensure_vector_index(self) -> None:
+        from google.cloud import firestore_admin_v1 as fa
+
+        admin = fa.FirestoreAdminClient()
+        parent = f"projects/{self._project}/databases/{self._database}/collectionGroups/{self._collection}"
+        for ix in admin.list_indexes(parent=parent):
+            paths = [f.field_path for f in ix.fields]
+            if "embedding" in paths and "scope" in paths:
+                return
+        index = fa.Index(
+            query_scope=fa.Index.QueryScope.COLLECTION,
+            fields=[
+                fa.Index.IndexField(field_path="scope", order=fa.Index.IndexField.Order.ASCENDING),
+                fa.Index.IndexField(
+                    field_path="embedding",
+                    vector_config=fa.Index.IndexField.VectorConfig(
+                        dimension=self.dimensions, flat=fa.Index.IndexField.VectorConfig.FlatIndex()
+                    ),
+                ),
+            ],
+        )
+        try:
+            admin.create_index(parent=parent, index=index)
+            logger.info("FirestoreMemoryBackend: creating vector index on %s (async)", self._collection)
+        except Exception as exc:  # already exists / racing creation
+            logger.info("FirestoreMemoryBackend: vector index create skipped: %s", exc)
+
+    # ------------------------------------------------------------------ helpers
+    @staticmethod
+    def _row(d: dict) -> dict:
+        row = {f: d.get(f) for f in ROW_FIELDS}
+        row["scope"] = norm_scope(d.get("scope"))
+        emb = d.get("embedding")
+        row["embedding"] = list(emb.to_map_value()["value"]) if isinstance(emb, Vector) else (list(emb) if emb else None)
+        return row
+
+    def _scope_queries(self, scope_prefix: Optional[str]):
+        """Firestore has no OR-with-range on one field, so a subtree is two queries."""
+        p = norm_scope(scope_prefix)
+        if p == "/":
+            # Every scope starts with "/", so a range on `scope` selects all rows while
+            # still using the scope+embedding composite index (a filter-less
+            # find_nearest would need a separate embedding-only index).
+            return [self._col.where(filter=FieldFilter("scope", ">=", "/"))]
+        return [
+            self._col.where(filter=FieldFilter("scope", "==", p)),
+            self._col.where(filter=FieldFilter("scope", ">=", p + "/")).where(filter=FieldFilter("scope", "<", p + "/" + _HIGH)),
+        ]
+
+    # ------------------------------------------------------------------ primitives
+    def _put(self, rows: List[dict]) -> None:
+        batch = self._client.batch()
+        for row in rows:
+            doc = {f: row.get(f) for f in ROW_FIELDS if f != "embedding"}
+            doc["scope"] = norm_scope(row["scope"])
+            doc["categories"] = list(row.get("categories") or [])
+            doc["metadata"] = dict(row.get("metadata") or {})
+            doc["private"] = bool(row.get("private", False))
+            if row.get("embedding"):
+                doc["embedding"] = Vector(row["embedding"])
+            batch.set(self._col.document(row["id"]), doc)
+        batch.commit()
+
+    def _get(self, record_id: str) -> Optional[dict]:
+        snap = self._col.document(record_id).get()
+        return self._row(snap.to_dict()) if snap.exists else None
+
+    def _delete_ids(self, ids: List[str]) -> int:
+        n = 0
+        batch = self._client.batch()
+        for rid in ids:
+            ref = self._col.document(rid)
+            if ref.get().exists:
+                batch.delete(ref)
+                n += 1
+        batch.commit()
+        return n
+
+    def _scan(self, scope_prefix: Optional[str]) -> List[dict]:
+        rows: List[dict] = []
+        for q in self._scope_queries(scope_prefix):
+            rows.extend(self._row(d.to_dict()) for d in q.stream())
+        return rows
+
+    def _vector_search(
+        self, vector: List[float], scope_prefix: Optional[str], limit: int
+    ) -> List[Tuple[dict, float]]:
+        out: List[Tuple[dict, float]] = []
+        for q in self._scope_queries(scope_prefix):
+            vq = q.find_nearest(
+                vector_field="embedding", query_vector=Vector(vector),
+                distance_measure=DistanceMeasure.COSINE, limit=limit, distance_result_field="_distance",
+            )
+            for d in vq.stream():
+                data = d.to_dict()
+                dist = data.pop("_distance", None)
+                out.append((self._row(data), 1.0 - float(dist) if dist is not None else 0.0))
+        out.sort(key=lambda rs: -rs[1])
+        return out[:limit]

crewai_memory_firestore-0.1.0/tests/test_firestore_backend.py

@@ -0,0 +1,68 @@
+"""Firestore StorageBackend: the shared contract suite against a real Firestore database.
+
+The backend creates the composite vector index (scope ASC + embedding flat) via
+the Admin API; index builds are asynchronous so the fixture waits for READY and
+the collection name is fixed per DIMS so the index is reused across runs.
+"""
+import os
+import time
+
+import pytest
+
+from crewai_memory_core.contract import *  # noqa: F401,F403
+from crewai_memory_core.contract import DIMS
+from crewai_memory_firestore import FirestoreMemoryBackend
+
+PROJECT = os.environ.get("GCP_PROJECT") or os.environ.get("GOOGLE_CLOUD_PROJECT") or "gcdeveloper-new"
+COLLECTION = f"crewai_mem_test_d{DIMS}"
+
+
+def _available() -> bool:
+    try:
+        from google.cloud import firestore
+        firestore.Client(project=PROJECT).collection("x").limit(1).get()
+        return True
+    except Exception:
+        return False
+
+
+pytestmark = pytest.mark.skipif(not _available(), reason="Firestore not reachable")
+
+
+def _wait_index_ready(timeout: int = 600) -> None:
+    from google.cloud import firestore_admin_v1 as fa
+    admin = fa.FirestoreAdminClient()
+    parent = f"projects/{PROJECT}/databases/(default)/collectionGroups/{COLLECTION}"
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        for ix in admin.list_indexes(parent=parent):
+            paths = [f.field_path for f in ix.fields]
+            if "embedding" in paths and "scope" in paths and ix.state == fa.Index.State.READY:
+                return
+        time.sleep(10)
+    raise RuntimeError("vector index not READY in time")
+
+
+@pytest.fixture(scope="module")
+def _store():
+    b = FirestoreMemoryBackend(project_id=PROJECT, collection=COLLECTION, dimensions=DIMS)
+    _wait_index_ready()
+    yield b
+    b.reset()
+
+
+@pytest.fixture()
+def backend(_store):
+    _store.reset()
+    yield _store
+    _store.reset()
+
+
+def test_native_vector_path_used(backend, monkeypatch):
+    from crewai_memory_core.contract import _seed, EMB
+    _seed(backend)
+    called = []
+    orig = backend._vector_search
+    monkeypatch.setattr(backend, "_vector_search", lambda *a, **k: (called.append(1), orig(*a, **k))[1])
+    assert backend.search(EMB(["sushi"])[0], scope_prefix="/crew", limit=5)
+    assert called