simple-graph-retriever 1.0.0rc1__tar.gz

simple_graph_retriever-1.0.0rc1/.github/workflows/release.yml

@@ -0,0 +1,33 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+      - release-candidate
+
+jobs:
+  release:
+    permissions:
+      contents: write
+      issues: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install wheel setuptools build
+          npm install -g semantic-release semantic-release-pypi @semantic-release/github @semantic-release/git semantic-release-replace-plugin
+      - name: Clean dist directory
+        run: rm -rf dist/
+      - name: Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+          PYPI_REPOSITORY_URL: https://test.pypi.org/legacy/
+        run: npx semantic-release

simple_graph_retriever-1.0.0rc1/.gitignore

@@ -0,0 +1,6 @@
+.env*
+dist/
+*.cql
+*.ttl
+*.pdf
+examples/save_neo4j

simple_graph_retriever-1.0.0rc1/.releaserc.json

@@ -0,0 +1,62 @@
+{
+  "branches": [
+    "main",
+    {
+      "name": "release-candidate",
+      "prerelease": "rc"
+    }
+  ],
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+    [
+      "semantic-release-pypi",
+      {
+        "repoUrl": "https://upload.pypi.org/legacy/"
+      }
+    ],
+    [
+      "@semantic-release/github",
+      {
+        "assets": [
+          {
+            "path": "dist/*.whl",
+            "label": "Wheel distribution"
+          },
+          {
+            "path": "dist/*.tar.gz",
+            "label": "Source distribution"
+          }
+        ]
+      }
+    ],
+    [
+      "semantic-release-replace-plugin",
+      {
+        "replacements": [
+          {
+            "files": ["README.md"],
+            "from": "pip install simple_graph_retriever==.*",
+            "to": "pip install simple_graph_retriever==${nextRelease.version}",
+            "results": [
+              {
+                "file": "README.md",
+                "hasChanged": true,
+                "numMatches": 1,
+                "numReplacements": 1
+              }
+            ],
+            "countMatches": true
+          }
+        ]
+      }
+    ],
+    [
+      "@semantic-release/git",
+      {
+        "assets": ["pyproject.toml", "README.md"],
+        "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
+      }
+    ]
+  ]
+}

simple_graph_retriever-1.0.0rc1/PKG-INFO

@@ -0,0 +1,203 @@
+Metadata-Version: 2.4
+Name: simple-graph-retriever
+Version: 1.0.0rc1
+Summary: A simple graph-based retriever using Neo4j and Qdrant
+Author-email: Victor de La Salmonière <victor@lettria.com>
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Requires-Dist: igraph
+Requires-Dist: leidenalg
+Requires-Dist: neo4j
+Requires-Dist: pydantic
+Requires-Dist: pydantic-settings
+Requires-Dist: qdrant-client
+Requires-Dist: requests
+Requires-Dist: tqdm
+Description-Content-Type: text/markdown
+
+# Simple Graph Retriever
+
+A Python SDK for indexing a Neo4j graph into a vector database (Qdrant) and performing retrieval-augmented generation (RAG) tasks against it.
+
+This SDK handles the heavy lifting of graph processing, including:
+
+- **Community Detection**: Uses the Leiden algorithm to identify communities within your graph structure.
+- **Chunking**: Breaks down nodes and their local context into "chunks" suitable for embedding.
+- **Vector Indexing**: Embeds and indexes both community summaries and individual chunks into Qdrant.
+- **Graph Retrieval**: Retrieves relevant subgraphs based on a natural language query, with fine-grained control over the expansion process.
+
+---
+
+## Table of Contents
+
+- [Simple Graph Retriever](#simple-graph-retriever)
+  - [Table of Contents](#table-of-contents)
+  - [1. Installation](#1-installation)
+  - [2. Configuration](#2-configuration)
+  - [3. Usage](#3-usage)
+    - [Initializing the Client](#initializing-the-client)
+    - [Indexing the Graph](#indexing-the-graph)
+    - [Retrieving a Subgraph](#retrieving-a-subgraph)
+      - [Basic Retrieval](#basic-retrieval)
+      - [Advanced Retrieval with Graph Expansion Control](#advanced-retrieval-with-graph-expansion-control)
+  - [4. Project Structure](#4-project-structure)
+  - [5. Contributing](#5-contributing)
+  - [6. License](#6-license)
+
+---
+
+## 1. Installation
+
+To install the SDK, clone this repository and use pip to install it in editable mode, which is recommended for development.
+
+```bash
+pip install simple_graph_retriever==0.0.0
+```
+
+## 2. Configuration
+
+The SDK is configured via environment variables, which are loaded from a `.env` file in your project's root directory.
+
+| Variable                        | Default Value           | Description                                                                                       |
+| ------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------- |
+| `NEO4J_URI`                     | `bolt://localhost:7687` | The URI for your Neo4j database.                                                                  |
+| `NEO4J_USER`                    | `neo4j`                 | The username for your Neo4j database.                                                             |
+| `NEO4J_PASSWORD`                | `password`              | The password for your Neo4j database.                                                             |
+| `QDRANT_URL`                    | `http://localhost:6333` | The URL for your Qdrant vector database instance.                                                 |
+| `QDRANT_API_KEY`                | `None`                  | Optional: The API key for authenticating with your Qdrant instance.                               |
+| `QDRANT_CHUNKS_COLLECTION`      | `chunks`                | The name of the collection for storing graph chunks.                                              |
+| `QDRANT_COMMUNITIES_COLLECTION` | `communities`           | The name of the collection for storing community summaries.                                       |
+| `EMBEDDER_URL`                  | `http://localhost:8080` | The URL of the text embedding service. It must accept a POST request with `{"inputs": ["text"]}`. |
+| `VECTOR_SIZE`                   | `384`                   | The dimension of the vectors produced by your embedding model.                                    |
+| `LOGLEVEL`                      | `INFO`                  | The logging level for the SDK (`DEBUG`, `INFO`, `WARNING`, `ERROR`).                              |
+
+**Example `.env` file:**
+
+```dotenv
+# Neo4j Database Credentials
+NEO4J_URI="bolt://localhost:7687"
+NEO4J_USER="neo4j"
+NEO4J_PASSWORD="your_secure_password"
+
+# Qdrant Vector Database URL
+QDRANT_URL="http://localhost:6333"
+QDRANT_API_KEY="your_qdrant_api_key" # Optional
+QDRANT_CHUNKS_COLLECTION="chunks"
+QDRANT_COMMUNITIES_COLLECTION="communities"
+
+# Text Embedding Model URL
+EMBEDDER_URL="http://localhost:8080/embed"
+```
+
+## 3. Usage
+
+### Initializing the Client
+
+The main entry point is the `GraphRetrievalClient`. It automatically loads settings from your `.env` file.
+
+```python
+from graph_retrieval_sdk.client import GraphRetrievalClient
+
+# Initialize the client
+client = GraphRetrievalClient()
+```
+
+### Indexing the Graph
+
+The `index()` method runs the full, idempotent pipeline to populate Qdrant with your graph data.
+
+```python
+# This runs the full pipeline:
+# 1. Community detection
+# 2. Chunk creation
+# 3. Chunk and Community indexing
+client.index()
+
+print("Indexing complete!")
+```
+
+To completely reset your vector index, use the `clear_index()` method. **Warning**: This is a destructive operation that deletes and recreates the Qdrant collections.
+
+```python
+client.clear_index()
+```
+
+### Retrieving a Subgraph
+
+Use the `retrieve_graph()` method to query your indexed graph. The `RetrievalConfig` model allows for fine-grained control over the process.
+
+#### Basic Retrieval
+
+```python
+from graph_retrieval_sdk.models import RetrievalConfig
+import json
+
+query = "Who was the emperor of Rome?"
+config = RetrievalConfig()
+
+subgraph = client.retriever.retrieve_graph(query, config)
+
+if subgraph:
+    print(f"Retrieved {len(subgraph[0]['nodes'])} nodes and {len(subgraph[0]['relationships'])} relationships.")
+```
+
+#### Advanced Retrieval with Graph Expansion Control
+
+Customize the `RetrievalConfig` to tune the retrieval and expansion behavior.
+
+```python
+from graph_retrieval_sdk.models import RetrievalConfig
+
+query = "Tell me about the Roman emperors and their families, but exclude servants."
+
+advanced_config = RetrievalConfig(
+    # --- Retrieval Settings ---
+    max_communities=5,
+    max_chunks=15,
+
+    # --- Graph Expansion Settings ---
+    max_hops=2,
+    community_expansion_limit=5,
+
+    # --- Relationship Filtering ---
+    allowed_rel_types=["HAS_SON", "MARRIED_TO", "SUCCESSOR_OF"],
+    denied_rel_types=["HAS_SERVANT"]
+)
+
+subgraph = client.retriever.retrieve_graph(query, advanced_config)
+
+if subgraph:
+    print(f"Retrieved {len(subgraph[0]['nodes'])} nodes with advanced configuration.")
+
+# Close the client connection when done
+client.close()
+```
+
+## 4. Project Structure
+
+The project is organized as a standard Python package:
+
+```
+├── graph_retrieval_sdk/
+│   ├── __init__.py
+│   ├── client.py         # Main client class
+│   ├── config.py         # Configuration and logging setup
+│   ├── indexer.py        # Logic for indexing the graph
+│   ├── retriever.py      # Logic for retrieving subgraphs
+│   └── models.py         # Pydantic models (e.g., RetrievalConfig)
+├── examples/
+│   ├── retrieve_graph.py
+│   └── run_indexing.py
+├── pyproject.toml        # Project metadata and dependencies
+└── README.md             # This file
+```
+
+## 5. Contributing
+
+Contributions are welcome! Please feel free to submit a pull request or open an issue.
+
+## 6. License
+
+This project is licensed under the MIT License.

simple_graph_retriever-1.0.0rc1/README.md

@@ -0,0 +1,184 @@
+# Simple Graph Retriever
+
+A Python SDK for indexing a Neo4j graph into a vector database (Qdrant) and performing retrieval-augmented generation (RAG) tasks against it.
+
+This SDK handles the heavy lifting of graph processing, including:
+
+- **Community Detection**: Uses the Leiden algorithm to identify communities within your graph structure.
+- **Chunking**: Breaks down nodes and their local context into "chunks" suitable for embedding.
+- **Vector Indexing**: Embeds and indexes both community summaries and individual chunks into Qdrant.
+- **Graph Retrieval**: Retrieves relevant subgraphs based on a natural language query, with fine-grained control over the expansion process.
+
+---
+
+## Table of Contents
+
+- [Simple Graph Retriever](#simple-graph-retriever)
+  - [Table of Contents](#table-of-contents)
+  - [1. Installation](#1-installation)
+  - [2. Configuration](#2-configuration)
+  - [3. Usage](#3-usage)
+    - [Initializing the Client](#initializing-the-client)
+    - [Indexing the Graph](#indexing-the-graph)
+    - [Retrieving a Subgraph](#retrieving-a-subgraph)
+      - [Basic Retrieval](#basic-retrieval)
+      - [Advanced Retrieval with Graph Expansion Control](#advanced-retrieval-with-graph-expansion-control)
+  - [4. Project Structure](#4-project-structure)
+  - [5. Contributing](#5-contributing)
+  - [6. License](#6-license)
+
+---
+
+## 1. Installation
+
+To install the SDK, clone this repository and use pip to install it in editable mode, which is recommended for development.
+
+```bash
+pip install simple_graph_retriever==0.0.0
+```
+
+## 2. Configuration
+
+The SDK is configured via environment variables, which are loaded from a `.env` file in your project's root directory.
+
+| Variable                        | Default Value           | Description                                                                                       |
+| ------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------- |
+| `NEO4J_URI`                     | `bolt://localhost:7687` | The URI for your Neo4j database.                                                                  |
+| `NEO4J_USER`                    | `neo4j`                 | The username for your Neo4j database.                                                             |
+| `NEO4J_PASSWORD`                | `password`              | The password for your Neo4j database.                                                             |
+| `QDRANT_URL`                    | `http://localhost:6333` | The URL for your Qdrant vector database instance.                                                 |
+| `QDRANT_API_KEY`                | `None`                  | Optional: The API key for authenticating with your Qdrant instance.                               |
+| `QDRANT_CHUNKS_COLLECTION`      | `chunks`                | The name of the collection for storing graph chunks.                                              |
+| `QDRANT_COMMUNITIES_COLLECTION` | `communities`           | The name of the collection for storing community summaries.                                       |
+| `EMBEDDER_URL`                  | `http://localhost:8080` | The URL of the text embedding service. It must accept a POST request with `{"inputs": ["text"]}`. |
+| `VECTOR_SIZE`                   | `384`                   | The dimension of the vectors produced by your embedding model.                                    |
+| `LOGLEVEL`                      | `INFO`                  | The logging level for the SDK (`DEBUG`, `INFO`, `WARNING`, `ERROR`).                              |
+
+**Example `.env` file:**
+
+```dotenv
+# Neo4j Database Credentials
+NEO4J_URI="bolt://localhost:7687"
+NEO4J_USER="neo4j"
+NEO4J_PASSWORD="your_secure_password"
+
+# Qdrant Vector Database URL
+QDRANT_URL="http://localhost:6333"
+QDRANT_API_KEY="your_qdrant_api_key" # Optional
+QDRANT_CHUNKS_COLLECTION="chunks"
+QDRANT_COMMUNITIES_COLLECTION="communities"
+
+# Text Embedding Model URL
+EMBEDDER_URL="http://localhost:8080/embed"
+```
+
+## 3. Usage
+
+### Initializing the Client
+
+The main entry point is the `GraphRetrievalClient`. It automatically loads settings from your `.env` file.
+
+```python
+from graph_retrieval_sdk.client import GraphRetrievalClient
+
+# Initialize the client
+client = GraphRetrievalClient()
+```
+
+### Indexing the Graph
+
+The `index()` method runs the full, idempotent pipeline to populate Qdrant with your graph data.
+
+```python
+# This runs the full pipeline:
+# 1. Community detection
+# 2. Chunk creation
+# 3. Chunk and Community indexing
+client.index()
+
+print("Indexing complete!")
+```
+
+To completely reset your vector index, use the `clear_index()` method. **Warning**: This is a destructive operation that deletes and recreates the Qdrant collections.
+
+```python
+client.clear_index()
+```
+
+### Retrieving a Subgraph
+
+Use the `retrieve_graph()` method to query your indexed graph. The `RetrievalConfig` model allows for fine-grained control over the process.
+
+#### Basic Retrieval
+
+```python
+from graph_retrieval_sdk.models import RetrievalConfig
+import json
+
+query = "Who was the emperor of Rome?"
+config = RetrievalConfig()
+
+subgraph = client.retriever.retrieve_graph(query, config)
+
+if subgraph:
+    print(f"Retrieved {len(subgraph[0]['nodes'])} nodes and {len(subgraph[0]['relationships'])} relationships.")
+```
+
+#### Advanced Retrieval with Graph Expansion Control
+
+Customize the `RetrievalConfig` to tune the retrieval and expansion behavior.
+
+```python
+from graph_retrieval_sdk.models import RetrievalConfig
+
+query = "Tell me about the Roman emperors and their families, but exclude servants."
+
+advanced_config = RetrievalConfig(
+    # --- Retrieval Settings ---
+    max_communities=5,
+    max_chunks=15,
+
+    # --- Graph Expansion Settings ---
+    max_hops=2,
+    community_expansion_limit=5,
+
+    # --- Relationship Filtering ---
+    allowed_rel_types=["HAS_SON", "MARRIED_TO", "SUCCESSOR_OF"],
+    denied_rel_types=["HAS_SERVANT"]
+)
+
+subgraph = client.retriever.retrieve_graph(query, advanced_config)
+
+if subgraph:
+    print(f"Retrieved {len(subgraph[0]['nodes'])} nodes with advanced configuration.")
+
+# Close the client connection when done
+client.close()
+```
+
+## 4. Project Structure
+
+The project is organized as a standard Python package:
+
+```
+├── graph_retrieval_sdk/
+│   ├── __init__.py
+│   ├── client.py         # Main client class
+│   ├── config.py         # Configuration and logging setup
+│   ├── indexer.py        # Logic for indexing the graph
+│   ├── retriever.py      # Logic for retrieving subgraphs
+│   └── models.py         # Pydantic models (e.g., RetrievalConfig)
+├── examples/
+│   ├── retrieve_graph.py
+│   └── run_indexing.py
+├── pyproject.toml        # Project metadata and dependencies
+└── README.md             # This file
+```
+
+## 5. Contributing
+
+Contributions are welcome! Please feel free to submit a pull request or open an issue.
+
+## 6. License
+
+This project is licensed under the MIT License.

simple_graph_retriever-1.0.0rc1/examples/index.py

@@ -0,0 +1,26 @@
+import sys
+import os
+
+from simple_graph_retriever.client import GraphRetrievalClient
+
+if __name__ == "__main__":
+    print("Starting graph indexing example...")
+    client = None
+    try:
+        # Initialize the client (settings loaded from .env or defaults)
+        client = GraphRetrievalClient()
+
+        # Run the full indexing pipeline
+        print(
+            "Running client.index()... (This will detect communities, create chunks, and index them)"
+        )
+        client.index()
+
+        print("✅ Graph indexing complete.")
+
+    except Exception as e:
+        print(f"❌ An error occurred during indexing: {e}")
+    finally:
+        if client:
+            client.close()
+            print("Client closed.")

simple_graph_retriever-1.0.0rc1/examples/retrieve.py

@@ -0,0 +1,55 @@
+from simple_graph_retriever.client import GraphRetrievalClient
+from simple_graph_retriever.models import RetrievalConfig
+import json
+
+if __name__ == "__main__":
+    client = GraphRetrievalClient()
+    try:
+        query = "Pizza"
+
+        print(f"Retrieving subgraph for query: '{query}'")
+
+        print("\n--- Full GraphRAG mode ---")
+        full_subgraph = client.retriever.retrieve_graph(
+            query,
+            RetrievalConfig(
+                use_communities=True,
+                use_chunks=True,
+            ),
+        )
+        if full_subgraph:
+            print(
+                f"Retrieved {len(full_subgraph[0]['nodes'])} nodes and {len(full_subgraph[0]['relationships'])} relationships."
+            )
+            with open("retrieved_subgraph_full.json", "w") as f:
+                json.dump(full_subgraph, f, indent=2)
+        else:
+            print("No results found.")
+
+        print("\n--- Chunk-only mode ---")
+        chunk_only_subgraph = client.retriever.retrieve_graph(
+            query, RetrievalConfig(use_communities=False, use_chunks=True)
+        )
+        if chunk_only_subgraph:
+            print(
+                f"Retrieved {len(chunk_only_subgraph[0]['nodes'])} nodes and {len(chunk_only_subgraph[0]['relationships'])} relationships."
+            )
+            with open("retrieved_subgraph_chunk_only.json", "w") as f:
+                json.dump(chunk_only_subgraph, f, indent=2)
+        else:
+            print("No results found.")
+
+        print("\n--- Community-only mode ---")
+        community_only_subgraph = client.retriever.retrieve_graph(
+            query, RetrievalConfig(use_communities=True, use_chunks=False)
+        )
+        if community_only_subgraph:
+            print(
+                f"Retrieved {len(community_only_subgraph[0]['nodes'])} nodes and {len(community_only_subgraph[0]['relationships'])} relationships."
+            )
+            with open("retrieved_subgraph_community_only.json", "w") as f:
+                json.dump(community_only_subgraph, f, indent=2)
+        else:
+            print("No results found.")
+    finally:
+        client.close()

simple_graph_retriever-1.0.0rc1/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "simple-graph-retriever"
+version = "1.0.0rc1"
+authors = [
+  { name="Victor de La Salmonière", email="victor@lettria.com" },
+]
+description = "A simple graph-based retriever using Neo4j and Qdrant"
+readme = "README.md"
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "neo4j",
+    "qdrant-client",
+    "requests",
+    "tqdm",
+    "igraph",
+    "leidenalg",
+    "pydantic",
+    "pydantic-settings",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["simple_graph_retriever"]

simple_graph_retriever-1.0.0rc1/simple_graph_retriever/client.py

@@ -0,0 +1,78 @@
+from typing import Optional, Tuple
+from .indexer import GraphIndexer
+from .retriever import GraphRetriever
+from .config import settings
+
+
+class GraphRetrievalClient:
+    """
+    A client for interacting with the graph retrieval API.
+    """
+
+    def __init__(self, 
+                 neo4j_uri: Optional[str] = None, 
+                 neo4j_auth: Optional[Tuple[str, str]] = None, 
+                 qdrant_url: Optional[str] = None, 
+                 qdrant_api_key: Optional[str] = None,
+                 qdrant_chunks_collection: Optional[str] = None,
+                 qdrant_communities_collection: Optional[str] = None,
+                 embedder_url: Optional[str] = None, 
+                 vector_size: Optional[int] = None):
+        """
+        Initializes the GraphRetrievalClient.
+        """
+        neo4j_uri = neo4j_uri or settings.neo4j_uri
+        neo4j_auth = neo4j_auth or (settings.neo4j_user, settings.neo4j_password)
+        qdrant_url = qdrant_url or settings.qdrant_url
+        qdrant_api_key = qdrant_api_key or settings.qdrant_api_key
+        qdrant_chunks_collection = qdrant_chunks_collection or settings.qdrant_chunks_collection
+        qdrant_communities_collection = qdrant_communities_collection or settings.qdrant_communities_collection
+        embedder_url = embedder_url or settings.embedder_url
+        vector_size = vector_size or settings.vector_size
+
+        self._indexer = GraphIndexer(
+            neo4j_uri=neo4j_uri, 
+            neo4j_auth=neo4j_auth, 
+            qdrant_url=qdrant_url, 
+            embedder_url=embedder_url, 
+            qdrant_api_key=qdrant_api_key, 
+            vector_size=vector_size,
+            chunks_collection=qdrant_chunks_collection,
+            communities_collection=qdrant_communities_collection
+        )
+        self._retriever = GraphRetriever(
+            qdrant_url=qdrant_url, 
+            neo4j_uri=neo4j_uri, 
+            neo4j_auth=neo4j_auth, 
+            embedder_url=embedder_url, 
+            qdrant_api_key=qdrant_api_key,
+            chunks_collection=qdrant_chunks_collection,
+            communities_collection=qdrant_communities_collection
+        )
+
+    @property
+    def indexer(self) -> GraphIndexer:
+        return self._indexer
+
+    @property
+    def retriever(self) -> GraphRetriever:
+        return self._retriever
+
+    def close(self):
+        self._indexer.close()
+        self._retriever.close()
+
+    def index(self):
+        """
+        Runs the complete graph indexing pipeline.
+        """
+        self.indexer.run_community_detection()
+        self.indexer.create_chunks()
+        self.indexer.index_chunks()
+        self.indexer.index_communities()
+
+    def clear_index(self):
+        """
+        Clears the Qdrant collections.
+        """
+        self.indexer.clear_qdrant_collections()

simple_graph_retriever-1.0.0rc1/simple_graph_retriever/config.py

@@ -0,0 +1,36 @@
+import logging
+import os
+import sys
+from typing import Optional
+from pydantic import ValidationError
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+logging.basicConfig(level=os.getenv("LOGLEVEL", "WARNING"))
+logger = logging.getLogger(__name__)
+
+
+class Settings(BaseSettings):
+    neo4j_uri: str = "bolt://localhost:7687"
+    neo4j_user: str = "neo4j"
+    neo4j_password: str = "password"
+    qdrant_url: str = "http://localhost:6333"
+    qdrant_api_key: Optional[str] = None
+    qdrant_chunks_collection: str = "chunks"
+    qdrant_communities_collection: str = "communities"
+    embedder_url: str = "http://localhost:8080"
+    vector_size: int = 384
+    loglevel: str = "WARNING"
+
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
+
+
+try:
+    settings = Settings()
+except ValidationError as e:
+    missing_fields = [
+        error["loc"][0] for error in e.errors() if error["type"] == "missing"
+    ]
+    logger.error(
+        f"Missing configuration fields: {', '.join(str(field) for field in missing_fields)}"
+    )
+    sys.exit(1)

simple_graph_retriever-1.0.0rc1/simple_graph_retriever/indexer.py

@@ -0,0 +1,367 @@
+import json
+import time
+from typing import List, Dict, Any, Optional
+from neo4j import GraphDatabase
+from qdrant_client import QdrantClient
+from qdrant_client.http.models import PointStruct, Distance, VectorParams
+import requests
+from tqdm import tqdm
+import igraph as ig
+import leidenalg as la
+from .config import logger
+
+
+class GraphIndexer:
+    def __init__(
+        self,
+        neo4j_uri,
+        neo4j_auth,
+        qdrant_url,
+        embedder_url,
+        qdrant_api_key: Optional[str] = None,
+        vector_size=384,
+        chunks_collection: str = "chunks",
+        communities_collection: str = "communities",
+    ):
+        self.driver = GraphDatabase.driver(neo4j_uri, auth=neo4j_auth)
+        self.qdrant = QdrantClient(url=qdrant_url, api_key=qdrant_api_key)
+        self.embedder_url = embedder_url
+        self.vector_batch_size = 100
+        self.vector_size = vector_size
+        self.chunks_collection = chunks_collection
+        self.communities_collection = communities_collection
+        self._ensure_qdrant_collections_exist()
+
+    def _ensure_qdrant_collections_exist(self):
+        logger.info("Ensuring Qdrant collections exist...")
+        existing_collections = [
+            c.name for c in self.qdrant.get_collections().collections
+        ]
+
+        if self.chunks_collection not in existing_collections:
+            self.qdrant.create_collection(
+                collection_name=self.chunks_collection,
+                vectors_config=VectorParams(
+                    size=self.vector_size, distance=Distance.COSINE
+                ),
+            )
+            logger.info(f"Created '{self.chunks_collection}' collection.")
+        else:
+            logger.info(f"'{self.chunks_collection}' collection already exists.")
+
+        if self.communities_collection not in existing_collections:
+            self.qdrant.create_collection(
+                collection_name=self.communities_collection,
+                vectors_config=VectorParams(
+                    size=self.vector_size, distance=Distance.COSINE
+                ),
+            )
+            logger.info(f"Created '{self.communities_collection}' collection.")
+        else:
+            logger.info(f"'{self.communities_collection}' collection already exists.")
+        logger.info("✅ Qdrant collections checked/created.")
+
+    def close(self):
+        self.driver.close()
+
+    def clear_qdrant_collections(self):
+        logger.info("🗑️ Clearing Qdrant collections...")
+        self.qdrant.recreate_collection(
+            collection_name=self.chunks_collection,
+            vectors_config=VectorParams(
+                size=self.vector_size, distance=Distance.COSINE
+            ),
+        )
+        self.qdrant.recreate_collection(
+            collection_name=self.communities_collection,
+            vectors_config=VectorParams(
+                size=self.vector_size, distance=Distance.COSINE
+            ),
+        )
+        logger.info("✅ Qdrant collections cleared and recreated.")
+
+    def embed_batch(self, texts: List[str]) -> List[List[float]]:
+        """
+        Batch embedding wrapper.
+        Adjust payload/response handling based on your specific API.
+        """
+        # Example assumes an OpenAI-compatible interface or similar list-in/list-out
+        embeddings = []
+        for text in texts:
+            try:
+                resp = requests.post(
+                    self.embedder_url,
+                    json={"inputs": [text]},  # Sending as a list of one string
+                    timeout=30,
+                )
+                resp.raise_for_status()
+                # Assuming response is a list of embeddings, even for single input
+                # e.g., [[0.1, 0.2, ...]]
+                embeddings.extend(
+                    resp.json()
+                )  # Use extend because response is list of lists
+            except Exception as e:
+                logger.error(f"Embedding failed for text '{text[:50]}...': {e}")
+                embeddings.append([])  # Append empty list for failed embeddings
+        return embeddings
+
+    def run_community_detection(self):
+        logger.info("1️⃣  Refreshing Community Structure...")
+        
+        # Wipe old community data from Qdrant
+        logger.info(f"   Wiping Qdrant collection: '{self.communities_collection}'...")
+        self.qdrant.recreate_collection(
+            collection_name=self.communities_collection,
+            vectors_config=VectorParams(
+                size=self.vector_size, distance=Distance.COSINE
+            ),
+        )
+
+        with self.driver.session() as session:
+            # Wipe old community data from Neo4j
+            logger.info("   Wiping community data from Neo4j...")
+            session.run("MATCH (c:Community) DETACH DELETE c")
+            session.run("MATCH (n) WHERE n.community_id IS NOT NULL REMOVE n.community_id")
+            logger.info("   Community data wiped.")
+
+            # Fetch all nodes and relationships from Neo4j (excluding chunks and communities)
+            logger.info("   Fetching graph data from Neo4j (excluding chunks and communities)...")
+            nodes_data = session.run("MATCH (n) WHERE NOT n:GraphChunk AND NOT n:Community RETURN elementId(n) as id").data()
+            rels_data = session.run(
+                "MATCH (a)-[r]->(b) WHERE NOT a:GraphChunk AND NOT a:Community AND NOT b:GraphChunk AND NOT b:Community RETURN elementId(r) as id, elementId(a) as source, elementId(b) as target"
+            ).data()
+
+            node_id_to_idx = {node["id"]: i for i, node in enumerate(nodes_data)}
+
+            # Create an igraph graph
+            g = ig.Graph(directed=True)
+            g.add_vertices(len(nodes_data))
+            g.vs["neo4j_id"] = [node["id"] for node in nodes_data]
+
+            edges = []
+            for rel in rels_data:
+                source_idx = node_id_to_idx.get(rel["source"])
+                target_idx = node_id_to_idx.get(rel["target"])
+                if source_idx is not None and target_idx is not None:
+                    edges.append((source_idx, target_idx))
+            g.add_edges(edges)
+
+            logger.info(
+                f"   Created igraph with {g.vcount()} vertices and {g.ecount()} edges."
+            )
+
+            # Run Leiden algorithm
+            logger.info("   Running Leiden algorithm...")
+            partition = la.find_partition(g, la.ModularityVertexPartition)
+            logger.info(f"✅ Detected {len(partition)} communities.")
+
+            # Write community IDs back to Neo4j
+            logger.info("   Writing community IDs to Neo4j...")
+            tx = session.begin_transaction()
+            batch_size = 1000
+            count = 0
+            for i, community_id in enumerate(partition.membership):
+                neo4j_node_id = g.vs[i]["neo4j_id"]
+                tx.run(
+                    "MATCH (n) WHERE elementId(n) = $neo4j_node_id SET n.community_id = $community_id",
+                    neo4j_node_id=neo4j_node_id,
+                    community_id=community_id,
+                )
+                count += 1
+                if count % batch_size == 0:
+                    tx.commit()
+                    tx = session.begin_transaction()
+            tx.commit()
+            logger.info(f"✅ Wrote community IDs for {count} nodes.")
+
+            # Materialize Community Nodes (Optional but good for linkage)
+            session.run(
+                """
+                MATCH (n) WHERE n.community_id IS NOT NULL
+                WITH n.community_id AS cid, count(n) as size
+                MERGE (c:Community {id: cid})
+                SET c.size = size
+            """
+            )
+            logger.info("✅ Materialized :Community nodes.")
+
+    def create_chunks(self):
+        logger.info("2️⃣  Creating GraphChunks (Node + Context).")
+        query = """
+        MATCH (n)
+        WHERE NOT n:GraphChunk AND NOT n:Community
+        // Ensure we only create a chunk if one doesn't already exist for the node
+        AND NOT (n)<-[:CENTERED_ON]-(:GraphChunk)
+        
+        // This query will now run once and create all necessary chunks
+        WITH n 
+
+        // 1. Gather properties
+        WITH n,
+             apoc.text.join(labels(n), ", ") as lbls,
+             apoc.convert.toJson(properties(n)) as props
+
+        // 2. Gather 1-hop context (Relationships + Neighbor Labels)
+        CALL {
+            WITH n
+            MATCH (n)-[r]-(m)
+            WITH type(r) as rel_type, labels(m) as n_labels, m
+            RETURN collect(
+                rel_type + " -> " + coalesce(m.name, m.title, head(labels(m)), "Node")
+            )[0..10] as context_list // Capping context to 10 neighbors
+        }
+
+        // 3. Format Text Blob
+        WITH n, lbls, props,
+             "Node: " + lbls + "\nProps: " + props + "\nContext:\n" +
+             apoc.text.join(context_list, "\n") as chunk_text
+
+        // 4. Create Chunk Node
+        CREATE (c:GraphChunk {
+            id: randomUUID(),
+            community_id: n.community_id,
+            text: chunk_text
+        })
+        CREATE (c)-[:CENTERED_ON]->(n)
+        RETURN count(c) as created_count
+        """
+
+        with self.driver.session() as session:
+            result = session.run(query).single()
+            count = result["created_count"] if result else 0
+            logger.info(f"✅ Total chunks created: {count}")
+
+    def index_chunks(self):
+        logger.info("3️⃣  Indexing Chunks to Qdrant...")
+
+        # Read chunks that are not yet indexed
+        fetch_query = """
+        MATCH (c:GraphChunk)-[:CENTERED_ON]->(n)
+        WHERE c.indexed IS NULL
+        RETURN c.id as id, c.text as text, c.community_id as comm_id,
+               elementId(n) as center_node_id
+        LIMIT $batch_size
+        """
+
+        mark_done_query = """
+        MATCH (c:GraphChunk) WHERE c.id IN $ids
+        SET c.indexed = true
+        """
+
+        with self.driver.session() as session:
+            while True:
+                # 1. Fetch Batch from Neo4j
+                records = session.run(
+                    fetch_query, batch_size=self.vector_batch_size
+                ).data()
+                if not records:
+                    break
+
+                texts = [r["text"] for r in records]
+                ids = [r["id"] for r in records]
+
+                # 2. Embed
+                vectors = self.embed_batch(texts)
+
+                # 3. Prepare Points
+                points = []
+                for i, rec in enumerate(records):
+                    if not vectors[i]:
+                        logger.warning(
+                            f"Skipping chunk {rec['id']} due to failed embedding."
+                        )
+                        continue  # skip failed embeddings
+
+                    points.append(
+                        PointStruct(
+                            id=rec["id"],  # Using UUID from Neo4j
+                            vector=vectors[i],
+                            payload={
+                                "text": rec["text"],
+                                "community_id": rec["comm_id"],
+                                "center_node_id": rec[
+                                    "center_node_id"
+                                ],  # Store internal ID for retrieval
+                                "type": "chunk",
+                            },
+                        )
+                    )
+
+                # 4. Upsert to Qdrant
+                if points:
+                    self.qdrant.upsert(
+                        collection_name=self.chunks_collection, points=points
+                    )
+
+                # 5. Mark as done in Neo4j
+                session.run(mark_done_query, ids=ids)
+                logger.info(f"   Indexed {len(points)} chunks.")
+
+        logger.info("✅ Chunk indexing complete.")
+
+    def index_communities(self):
+        logger.info("4️⃣  Indexing Communities...")
+
+        # Strategy: Aggregate the text of the 5 most connected nodes in the community
+        query = """
+        MATCH (c:Community)
+        WHERE c.indexed IS NULL
+
+        CALL {
+            WITH c
+            MATCH (n) WHERE n.community_id = c.id
+            // Heuristic: Pick 'important' nodes by degree
+            WITH n ORDER BY COUNT { (n)--() } DESC LIMIT 5
+
+            // Re-generate basic text for these nodes
+            WITH n, labels(n)[0] + ": " + coalesce(n.name, n.title, "Item") as summary
+            RETURN collect(summary) as summaries
+        }
+
+        WITH c, "Community ID: " + toString(c.id) + "\nKey Elements:\n" + apoc.text.join(summaries, "\n") as comm_text
+        RETURN c.id as id, comm_text as text
+        LIMIT $batch_size
+        """
+
+        mark_done = "MATCH (c:Community {id: $id}) SET c.indexed = true"
+
+        with self.driver.session() as session:
+            while True:
+                records = session.run(query, batch_size=self.vector_batch_size).data()
+                if not records:
+                    break
+
+                texts = [r["text"] for r in records]
+                vectors = self.embed_batch(texts)
+
+                points = []
+                for i, rec in enumerate(records):
+                    if not vectors[i]:
+                        continue
+
+                    # Qdrant requires integer or UUID for ID.
+                    # If community_id is integer, use it directly.
+                    # If it's a string/uuid, you might need to hash it to UUID.
+                    points.append(
+                        PointStruct(
+                            id=rec["id"],
+                            vector=vectors[i],
+                            payload={
+                                "text": rec["text"],
+                                "community_id": rec["id"],
+                                "type": "community",
+                            },
+                        )
+                    )
+
+                if points:
+                    self.qdrant.upsert(
+                        collection_name=self.communities_collection, points=points
+                    )
+
+                for r in records:
+                    session.run(mark_done, id=r["id"])
+
+                logger.info(f"   Indexed {len(points)} communities.")
+
+        logger.info("✅ Community indexing complete.")

simple_graph_retriever-1.0.0rc1/simple_graph_retriever/models.py

@@ -0,0 +1,16 @@
+from typing import List, Optional
+from pydantic import BaseModel
+
+class RetrievalConfig(BaseModel):
+    use_communities: bool = True
+    use_chunks: bool = True
+    max_communities: int = 3
+    max_chunks: int = 10
+    max_hops: int = 1
+    community_score_threshold: float = 0.4
+    chunk_score_threshold: float = 0.4
+    community_score_drop_off_pct: float = 0.07
+    chunk_score_drop_off_pct: float = 0.2
+    community_expansion_limit: int = 10
+    allowed_rel_types: Optional[List[str]] = None
+    denied_rel_types: Optional[List[str]] = None

simple_graph_retriever-1.0.0rc1/simple_graph_retriever/retriever.py

@@ -0,0 +1,244 @@
+from typing import List, Optional
+import requests
+from qdrant_client import QdrantClient
+from qdrant_client.http.models import Filter, FieldCondition, MatchAny
+from neo4j import GraphDatabase
+
+from .models import RetrievalConfig
+from .config import logger
+
+
+class GraphRetriever:
+    def __init__(
+        self,
+        qdrant_url,
+        neo4j_uri,
+        neo4j_auth,
+        embedder_url,
+        qdrant_api_key: Optional[str] = None,
+        chunks_collection: str = "chunks",
+        communities_collection: str = "communities",
+    ):
+        self.qdrant_client = QdrantClient(url=qdrant_url, api_key=qdrant_api_key)
+        self.neo4j_driver = GraphDatabase.driver(neo4j_uri, auth=neo4j_auth)
+        self.embedder_url = embedder_url
+        self.chunks_collection = chunks_collection
+        self.communities_collection = communities_collection
+
+    def close(self):
+        self.neo4j_driver.close()
+
+    def embed(self, text: str) -> list[float]:
+        """
+        Embeds a single string using the TEI embedder.
+        """
+        try:
+            resp = requests.post(self.embedder_url, json={"inputs": text}, timeout=5)
+            resp.raise_for_status()
+            return resp.json()[0]
+        except Exception as e:
+            logger.error(f"Embedding failed: {e}")
+            return []
+
+    def search_qdrant(
+        self,
+        collection: str,
+        vector: list[float],
+        limit: int,
+        filters=None,
+        score_threshold=None,
+        score_drop_off_pct=None,
+    ):
+        try:
+            search_result = self.qdrant_client.search(
+                collection_name=collection,
+                query_vector=vector,
+                limit=limit,
+                query_filter=filters,
+                score_threshold=score_threshold,
+            )
+
+            # Apply score drop-off filter
+            if score_drop_off_pct is not None and len(search_result) > 1:
+                filtered_results = [search_result[0]]
+                for i in range(1, len(search_result)):
+                    prev_score = search_result[i - 1].score
+                    current_score = search_result[i].score
+
+                    if prev_score == 0:
+                        break
+
+                    drop_off = (prev_score - current_score) / prev_score
+                    if drop_off > score_drop_off_pct:
+                        break  # Stop including results after a significant drop-off
+                    filtered_results.append(search_result[i])
+                return filtered_results
+
+            return search_result
+        except Exception as e:
+            logger.error(f"Qdrant search failed: {e}")
+            return []
+
+    def retrieve_communities(self, query_embedding, config: RetrievalConfig):
+        if not config.use_communities:
+            return []
+
+        results = self.search_qdrant(
+            collection=self.communities_collection,
+            vector=query_embedding,
+            limit=config.max_communities,
+            score_threshold=config.community_score_threshold,
+            score_drop_off_pct=config.community_score_drop_off_pct,
+        )
+        return [
+            {"community_id": r.payload["community_id"], "score": r.score}
+            for r in results
+        ]
+
+    def retrieve_chunks(
+        self, query_embedding, community_results: list, config: RetrievalConfig
+    ):
+        if not config.use_chunks:
+            return []
+
+        filters = None
+        if community_results:
+            community_ids = [c["community_id"] for c in community_results]
+            filters = Filter(
+                must=[
+                    FieldCondition(
+                        key="community_id", match=MatchAny(any=community_ids)
+                    )
+                ]
+            )
+
+        results = self.search_qdrant(
+            collection=self.chunks_collection,
+            vector=query_embedding,
+            limit=config.max_chunks,
+            filters=filters,
+            score_threshold=config.chunk_score_threshold,
+            score_drop_off_pct=config.chunk_score_drop_off_pct,
+        )
+
+        return [
+            {"center_node_id": r.payload["center_node_id"], "score": r.score}
+            for r in results
+        ]
+
+    def retrieve_nodes_from_communities(
+        self, community_ids: list, community_expansion_limit: int
+    ):
+        query = """
+            MATCH (n)
+            WHERE n.community_id IN $cids
+            RETURN elementId(n) as id
+            LIMIT $limit
+        """
+        with self.neo4j_driver.session() as session:
+            result = session.run(
+                query, cids=community_ids, limit=community_expansion_limit
+            )
+            return [record["id"] for record in result]
+
+    def fetch_subgraph(
+        self,
+        center_node_ids: list,
+        max_hops: int,
+        allowed_rel_types: Optional[List[str]] = None,
+        denied_rel_types: Optional[List[str]] = None,
+    ):
+        rel_type_filter_clauses = []
+        if allowed_rel_types:
+            rel_type_filter_clauses.append("type(r) IN $allowed_rel_types")
+        if denied_rel_types:
+            rel_type_filter_clauses.append("NOT type(r) IN $denied_rel_types")
+
+        rel_filter_string = ""
+        if rel_type_filter_clauses:
+            rel_filter_string = " WHERE " + " AND ".join(rel_type_filter_clauses)
+
+        query = f"""
+            MATCH (n)
+            WHERE elementId(n) IN $ids
+            MATCH p=(n)-[r*1..{max_hops}]-(m)
+            {rel_filter_string}
+            UNWIND nodes(p) as node
+            UNWIND relationships(p) as rel
+            RETURN collect(DISTINCT node {{.*, element_id: elementId(node)}}) as nodes,
+                   collect(DISTINCT rel {{.*, element_id: elementId(rel), type: type(rel), start_node_element_id: elementId(startNode(rel)), end_node_element_id: elementId(endNode(rel))}}) as relationships
+        """
+        with self.neo4j_driver.session() as session:
+            params = {"ids": center_node_ids}
+            if allowed_rel_types:
+                params["allowed_rel_types"] = allowed_rel_types
+            if denied_rel_types:
+                params["denied_rel_types"] = denied_rel_types
+
+            result = session.run(query, params)
+            return result.data()
+
+    def retrieve_graph(
+        self, query: str, config: RetrievalConfig, include_chunks: bool = False
+    ):
+        """
+        Retrieves a subgraph from the graph database based on a query string.
+        """
+        query_embedding = self.embed(query)
+        if not query_embedding:
+            return []
+
+        community_results = self.retrieve_communities(query_embedding, config)
+        community_scores = {c["community_id"]: c["score"] for c in community_results}
+
+        chunk_results = []
+        center_node_ids = []
+        if config.use_chunks:
+            chunk_results = self.retrieve_chunks(
+                query_embedding, community_results, config
+            )
+            center_node_ids += [c["center_node_id"] for c in chunk_results]
+        if config.use_communities:
+            community_ids = [c["community_id"] for c in community_results]
+            center_node_ids += self.retrieve_nodes_from_communities(
+                community_ids,
+                community_expansion_limit=config.community_expansion_limit,
+            )
+
+        if not center_node_ids:
+            return []
+        # print(f"Number of center nodes: {len(center_node_ids)}")
+        subgraph = self.fetch_subgraph(
+            center_node_ids,
+            config.max_hops,
+            config.allowed_rel_types,
+            config.denied_rel_types,
+        )
+        # sub_graph_ids = [node["element_id"] for node in subgraph[0]["nodes"]]
+        # print(f"Subgraph node IDs: {sub_graph_ids}")
+        if subgraph and subgraph[0].get("nodes"):
+            chunk_scores = {c["center_node_id"]: c["score"] for c in chunk_results}
+
+            for node in subgraph[0]["nodes"]:
+                node_id = node.get("element_id")
+                community_id = node.get("community_id")
+                if node_id in chunk_scores:
+                    node["chunk_score"] = chunk_scores[node_id]
+                if community_id in community_scores:
+                    node["community_score"] = community_scores[community_id]
+
+        if not include_chunks:
+            nodes_to_keep = [
+                node for node in subgraph[0]["nodes"] if "text" not in node
+            ]
+            node_ids_to_keep = {node["element_id"] for node in nodes_to_keep}
+            relationships_to_keep = [
+                rel
+                for rel in subgraph[0]["relationships"]
+                if rel["start_node_element_id"] in node_ids_to_keep
+                and rel["end_node_element_id"] in node_ids_to_keep
+            ]
+            subgraph[0]["nodes"] = nodes_to_keep
+            subgraph[0]["relationships"] = relationships_to_keep
+
+        return subgraph