graphiti-core 0.2.3__py3-none-any.whl → 0.3.1__py3-none-any.whl

graphiti_core/search/search_utils.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024, Zep Software, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import asyncio
 import logging
 import re
@@ -7,7 +23,13 @@ from time import time
 from neo4j import AsyncDriver, Query
 
 from graphiti_core.edges import EntityEdge, get_entity_edge_from_record
-from graphiti_core.nodes import EntityNode, EpisodicNode, get_entity_node_from_record
+from graphiti_core.nodes import (
+    CommunityNode,
+    EntityNode,
+    EpisodicNode,
+    get_community_node_from_record,
+    get_entity_node_from_record,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +57,128 @@ async def get_mentioned_nodes(driver: AsyncDriver, episodes: list[EpisodicNode])
     return nodes
 
 
+async def edge_fulltext_search(
+    driver: AsyncDriver,
+    query: str,
+    source_node_uuid: str | None,
+    target_node_uuid: str | None,
+    group_ids: list[str | None] | None = None,
+    limit=RELEVANT_SCHEMA_LIMIT,
+) -> list[EntityEdge]:
+    # fulltext search over facts
+    cypher_query = Query("""
+              CALL db.index.fulltext.queryRelationships("name_and_fact", $query) 
+              YIELD relationship AS rel, score
+              MATCH (n:Entity {uuid: $source_uuid})-[r {uuid: rel.uuid}]-(m:Entity {uuid: $target_uuid})
+              WHERE CASE
+                WHEN $group_ids IS NULL THEN n.group_id IS NULL
+                ELSE n.group_id IN $group_ids
+              END
+              RETURN 
+                    r.uuid AS uuid,
+                    r.group_id AS group_id,
+                    n.uuid AS source_node_uuid,
+                    m.uuid AS target_node_uuid,
+                    r.created_at AS created_at,
+                    r.name AS name,
+                    r.fact AS fact,
+                    r.fact_embedding AS fact_embedding,
+                    r.episodes AS episodes,
+                    r.expired_at AS expired_at,
+                    r.valid_at AS valid_at,
+                    r.invalid_at AS invalid_at
+                ORDER BY score DESC LIMIT $limit
+                """)
+
+    if source_node_uuid is None and target_node_uuid is None:
+        cypher_query = Query("""
+                  CALL db.index.fulltext.queryRelationships("name_and_fact", $query) 
+                  YIELD relationship AS rel, score
+                  MATCH (n:Entity)-[r {uuid: rel.uuid}]-(m:Entity)
+                  WHERE CASE
+                    WHEN $group_ids IS NULL THEN r.group_id IS NULL
+                    ELSE r.group_id IN $group_ids
+                  END
+                  RETURN 
+                        r.uuid AS uuid,
+                        r.group_id AS group_id,
+                        n.uuid AS source_node_uuid,
+                        m.uuid AS target_node_uuid,
+                        r.created_at AS created_at,
+                        r.name AS name,
+                        r.fact AS fact,
+                        r.fact_embedding AS fact_embedding,
+                        r.episodes AS episodes,
+                        r.expired_at AS expired_at,
+                        r.valid_at AS valid_at,
+                        r.invalid_at AS invalid_at
+                    ORDER BY score DESC LIMIT $limit
+                    """)
+    elif source_node_uuid is None:
+        cypher_query = Query("""
+                  CALL db.index.fulltext.queryRelationships("name_and_fact", $query) 
+                  YIELD relationship AS rel, score
+                  MATCH (n:Entity)-[r {uuid: rel.uuid}]-(m:Entity {uuid: $target_uuid})              
+                  WHERE CASE
+                    WHEN $group_ids IS NULL THEN r.group_id IS NULL
+                    ELSE r.group_id IN $group_ids
+                  END
+                  RETURN 
+                        r.uuid AS uuid,
+                        r.group_id AS group_id,
+                        n.uuid AS source_node_uuid,
+                        m.uuid AS target_node_uuid,
+                        r.created_at AS created_at,
+                        r.name AS name,
+                        r.fact AS fact,
+                        r.fact_embedding AS fact_embedding,
+                        r.episodes AS episodes,
+                        r.expired_at AS expired_at,
+                        r.valid_at AS valid_at,
+                        r.invalid_at AS invalid_at
+                    ORDER BY score DESC LIMIT $limit
+                    """)
+    elif target_node_uuid is None:
+        cypher_query = Query("""
+                  CALL db.index.fulltext.queryRelationships("name_and_fact", $query) 
+                  YIELD relationship AS rel, score
+                  MATCH (n:Entity {uuid: $source_uuid})-[r {uuid: rel.uuid}]-(m:Entity)              
+                  WHERE CASE
+                    WHEN $group_ids IS NULL THEN r.group_id IS NULL
+                    ELSE r.group_id IN $group_ids
+                  END
+                  RETURN 
+                        r.uuid AS uuid,
+                        r.group_id AS group_id,
+                        n.uuid AS source_node_uuid,
+                        m.uuid AS target_node_uuid,
+                        r.created_at AS created_at,
+                        r.name AS name,
+                        r.fact AS fact,
+                        r.fact_embedding AS fact_embedding,
+                        r.episodes AS episodes,
+                        r.expired_at AS expired_at,
+                        r.valid_at AS valid_at,
+                        r.invalid_at AS invalid_at
+                    ORDER BY score DESC LIMIT $limit
+                    """)
+
+    fuzzy_query = re.sub(r'[^\w\s]', '', query) + '~'
+
+    records, _, _ = await driver.execute_query(
+        cypher_query,
+        query=fuzzy_query,
+        source_uuid=source_node_uuid,
+        target_uuid=target_node_uuid,
+        group_ids=group_ids,
+        limit=limit,
+    )
+
+    edges = [get_entity_edge_from_record(record) for record in records]
+
+    return edges
+
+
 async def edge_similarity_search(
     driver: AsyncDriver,
     search_vector: list[float],
@@ -43,13 +187,15 @@ async def edge_similarity_search(
     group_ids: list[str | None] | None = None,
     limit: int = RELEVANT_SCHEMA_LIMIT,
 ) -> list[EntityEdge]:
-    group_ids = group_ids if group_ids is not None else [None]
     # vector similarity search over embedded facts
     query = Query("""
                 CALL db.index.vector.queryRelationships("fact_embedding", $limit, $search_vector)
                 YIELD relationship AS rel, score
                 MATCH (n:Entity {uuid: $source_uuid})-[r {uuid: rel.uuid}]-(m:Entity {uuid: $target_uuid})
-                WHERE r.group_id IN $group_ids
+                WHERE CASE
+                    WHEN $group_ids IS NULL THEN r.group_id IS NULL
+                    ELSE r.group_id IN $group_ids
+                END
                 RETURN
                     r.uuid AS uuid,
                     r.group_id AS group_id,
@@ -71,7 +217,10 @@ async def edge_similarity_search(
                     CALL db.index.vector.queryRelationships("fact_embedding", $limit, $search_vector)
                     YIELD relationship AS rel, score
                     MATCH (n:Entity)-[r {uuid: rel.uuid}]-(m:Entity)
-                    WHERE r.group_id IN $group_ids
+                    WHERE CASE
+                        WHEN $group_ids IS NULL THEN r.group_id IS NULL
+                        ELSE r.group_id IN $group_ids
+                    END
                     RETURN
                         r.uuid AS uuid,
                         r.group_id AS group_id,
@@ -92,7 +241,10 @@ async def edge_similarity_search(
                     CALL db.index.vector.queryRelationships("fact_embedding", $limit, $search_vector)
                     YIELD relationship AS rel, score
                     MATCH (n:Entity)-[r {uuid: rel.uuid}]-(m:Entity {uuid: $target_uuid})
-                    WHERE r.group_id IN $group_ids
+                    WHERE CASE
+                        WHEN $group_ids IS NULL THEN r.group_id IS NULL
+                        ELSE r.group_id IN $group_ids
+                    END
                     RETURN
                         r.uuid AS uuid,
                         r.group_id AS group_id,
@@ -113,7 +265,10 @@ async def edge_similarity_search(
                     CALL db.index.vector.queryRelationships("fact_embedding", $limit, $search_vector)
                     YIELD relationship AS rel, score
                     MATCH (n:Entity {uuid: $source_uuid})-[r {uuid: rel.uuid}]-(m:Entity)
-                    WHERE r.group_id IN $group_ids
+                    WHERE CASE
+                        WHEN $group_ids IS NULL THEN r.group_id IS NULL
+                        ELSE r.group_id IN $group_ids
+                    END
                     RETURN
                         r.uuid AS uuid,
                         r.group_id AS group_id,
@@ -144,9 +299,44 @@ async def edge_similarity_search(
     return edges
 
 
-async def entity_similarity_search(
-    search_vector: list[float],
+async def node_fulltext_search(
     driver: AsyncDriver,
+    query: str,
+    group_ids: list[str | None] | None = None,
+    limit=RELEVANT_SCHEMA_LIMIT,
+) -> list[EntityNode]:
+    # BM25 search to get top nodes
+    fuzzy_query = re.sub(r'[^\w\s]', '', query) + '~'
+    records, _, _ = await driver.execute_query(
+        """
+    CALL db.index.fulltext.queryNodes("name_and_summary", $query) 
+    YIELD node AS n, score
+    WHERE CASE
+        WHEN $group_ids IS NULL THEN n.group_id IS NULL
+        ELSE n.group_id IN $group_ids
+    END
+    RETURN
+        n.uuid AS uuid,
+        n.group_id AS group_id, 
+        n.name AS name, 
+        n.name_embedding AS name_embedding,
+        n.created_at AS created_at, 
+        n.summary AS summary
+    ORDER BY score DESC
+    LIMIT $limit
+    """,
+        query=fuzzy_query,
+        group_ids=group_ids,
+        limit=limit,
+    )
+    nodes = [get_entity_node_from_record(record) for record in records]
+
+    return nodes
+
+
+async def node_similarity_search(
+    driver: AsyncDriver,
+    search_vector: list[float],
     group_ids: list[str | None] | None = None,
     limit=RELEVANT_SCHEMA_LIMIT,
 ) -> list[EntityNode]:
@@ -176,28 +366,28 @@ async def entity_similarity_search(
     return nodes
 
 
-async def entity_fulltext_search(
-    query: str,
+async def community_fulltext_search(
     driver: AsyncDriver,
+    query: str,
     group_ids: list[str | None] | None = None,
     limit=RELEVANT_SCHEMA_LIMIT,
-) -> list[EntityNode]:
+) -> list[CommunityNode]:
     group_ids = group_ids if group_ids is not None else [None]
 
-    # BM25 search to get top nodes
+    # BM25 search to get top communities
     fuzzy_query = re.sub(r'[^\w\s]', '', query) + '~'
     records, _, _ = await driver.execute_query(
         """
-    CALL db.index.fulltext.queryNodes("name_and_summary", $query) 
-    YIELD node AS n, score
-    MATCH (n WHERE n.group_id in $group_ids)
+    CALL db.index.fulltext.queryNodes("community_name", $query) 
+    YIELD node AS comm, score
+    MATCH (comm WHERE comm.group_id in $group_ids)
     RETURN
-        n.uuid AS uuid,
-        n.group_id AS group_id, 
-        n.name AS name, 
-        n.name_embedding AS name_embedding,
-        n.created_at AS created_at, 
-        n.summary AS summary
+        comm.uuid AS uuid,
+        comm.group_id AS group_id, 
+        comm.name AS name, 
+        comm.name_embedding AS name_embedding,
+        comm.created_at AS created_at, 
+        comm.summary AS summary
     ORDER BY score DESC
     LIMIT $limit
     """,
@@ -205,121 +395,41 @@ async def entity_fulltext_search(
         group_ids=group_ids,
         limit=limit,
     )
-    nodes = [get_entity_node_from_record(record) for record in records]
+    communities = [get_community_node_from_record(record) for record in records]
 
-    return nodes
+    return communities
 
 
-async def edge_fulltext_search(
+async def community_similarity_search(
     driver: AsyncDriver,
-    query: str,
-    source_node_uuid: str | None,
-    target_node_uuid: str | None,
+    search_vector: list[float],
     group_ids: list[str | None] | None = None,
     limit=RELEVANT_SCHEMA_LIMIT,
-) -> list[EntityEdge]:
+) -> list[CommunityNode]:
     group_ids = group_ids if group_ids is not None else [None]
 
-    # fulltext search over facts
-    cypher_query = Query("""
-              CALL db.index.fulltext.queryRelationships("name_and_fact", $query) 
-              YIELD relationship AS rel, score
-              MATCH (n:Entity {uuid: $source_uuid})-[r {uuid: rel.uuid}]-(m:Entity {uuid: $target_uuid})
-              WHERE r.group_id IN $group_ids
-              RETURN 
-                    r.uuid AS uuid,
-                    r.group_id AS group_id,
-                    n.uuid AS source_node_uuid,
-                    m.uuid AS target_node_uuid,
-                    r.created_at AS created_at,
-                    r.name AS name,
-                    r.fact AS fact,
-                    r.fact_embedding AS fact_embedding,
-                    r.episodes AS episodes,
-                    r.expired_at AS expired_at,
-                    r.valid_at AS valid_at,
-                    r.invalid_at AS invalid_at
-                ORDER BY score DESC LIMIT $limit
-                """)
-
-    if source_node_uuid is None and target_node_uuid is None:
-        cypher_query = Query("""
-                  CALL db.index.fulltext.queryRelationships("name_and_fact", $query) 
-                  YIELD relationship AS rel, score
-                  MATCH (n:Entity)-[r {uuid: rel.uuid}]-(m:Entity)
-                  WHERE r.group_id IN $group_ids
-                  RETURN 
-                        r.uuid AS uuid,
-                        r.group_id AS group_id,
-                        n.uuid AS source_node_uuid,
-                        m.uuid AS target_node_uuid,
-                        r.created_at AS created_at,
-                        r.name AS name,
-                        r.fact AS fact,
-                        r.fact_embedding AS fact_embedding,
-                        r.episodes AS episodes,
-                        r.expired_at AS expired_at,
-                        r.valid_at AS valid_at,
-                        r.invalid_at AS invalid_at
-                    ORDER BY score DESC LIMIT $limit
-                    """)
-    elif source_node_uuid is None:
-        cypher_query = Query("""
-                  CALL db.index.fulltext.queryRelationships("name_and_fact", $query) 
-                  YIELD relationship AS rel, score
-                  MATCH (n:Entity)-[r {uuid: rel.uuid}]-(m:Entity {uuid: $target_uuid})              
-                  WHERE r.group_id IN $group_ids
-                  RETURN 
-                        r.uuid AS uuid,
-                        r.group_id AS group_id,
-                        n.uuid AS source_node_uuid,
-                        m.uuid AS target_node_uuid,
-                        r.created_at AS created_at,
-                        r.name AS name,
-                        r.fact AS fact,
-                        r.fact_embedding AS fact_embedding,
-                        r.episodes AS episodes,
-                        r.expired_at AS expired_at,
-                        r.valid_at AS valid_at,
-                        r.invalid_at AS invalid_at
-                    ORDER BY score DESC LIMIT $limit
-                    """)
-    elif target_node_uuid is None:
-        cypher_query = Query("""
-                  CALL db.index.fulltext.queryRelationships("name_and_fact", $query) 
-                  YIELD relationship AS rel, score
-                  MATCH (n:Entity {uuid: $source_uuid})-[r {uuid: rel.uuid}]-(m:Entity)              
-                  WHERE r.group_id IN $group_ids
-                  RETURN 
-                        r.uuid AS uuid,
-                        r.group_id AS group_id,
-                        n.uuid AS source_node_uuid,
-                        m.uuid AS target_node_uuid,
-                        r.created_at AS created_at,
-                        r.name AS name,
-                        r.fact AS fact,
-                        r.fact_embedding AS fact_embedding,
-                        r.episodes AS episodes,
-                        r.expired_at AS expired_at,
-                        r.valid_at AS valid_at,
-                        r.invalid_at AS invalid_at
-                    ORDER BY score DESC LIMIT $limit
-                    """)
-
-    fuzzy_query = re.sub(r'[^\w\s]', '', query) + '~'
-
+    # vector similarity search over entity names
     records, _, _ = await driver.execute_query(
-        cypher_query,
-        query=fuzzy_query,
-        source_uuid=source_node_uuid,
-        target_uuid=target_node_uuid,
+        """
+                CALL db.index.vector.queryNodes("community_name_embedding", $limit, $search_vector)
+                YIELD node AS comm, score
+                MATCH (comm WHERE comm.group_id IN $group_ids)
+                RETURN
+                    comm.uuid As uuid,
+                    comm.group_id AS group_id,
+                    comm.name AS name, 
+                    comm.name_embedding AS name_embedding,
+                    comm.created_at AS created_at, 
+                    comm.summary AS summary
+                ORDER BY score DESC
+                """,
+        search_vector=search_vector,
         group_ids=group_ids,
         limit=limit,
     )
+    communities = [get_community_node_from_record(record) for record in records]
 
-    edges = [get_entity_edge_from_record(record) for record in records]
-
-    return edges
+    return communities
 
 
 async def hybrid_node_search(
@@ -371,8 +481,8 @@ async def hybrid_node_search(
 
     results: list[list[EntityNode]] = list(
         await asyncio.gather(
-            *[entity_fulltext_search(q, driver, group_ids, 2 * limit) for q in queries],
-            *[entity_similarity_search(e, driver, group_ids, 2 * limit) for e in embeddings],
+            *[node_fulltext_search(driver, q, group_ids, 2 * limit) for q in queries],
+            *[node_similarity_search(driver, e, group_ids, 2 * limit) for e in embeddings],
         )
     )
 
@@ -490,40 +600,37 @@ def rrf(results: list[list[str]], rank_const=1) -> list[str]:
 
 
 async def node_distance_reranker(
-    driver: AsyncDriver, results: list[list[str]], center_node_uuid: str
+    driver: AsyncDriver, node_uuids: list[list[str]], center_node_uuid: str
 ) -> list[str]:
     # use rrf as a preliminary ranker
-    sorted_uuids = rrf(results)
+    sorted_uuids = rrf(node_uuids)
     scores: dict[str, float] = {}
 
-    for uuid in sorted_uuids:
-        # Find the shortest path to center node
-        records, _, _ = await driver.execute_query(
-            """  
-        MATCH (source:Entity)-[r:RELATES_TO {uuid: $edge_uuid}]->(target:Entity)
-        MATCH p = SHORTEST 1 (center:Entity)-[:RELATES_TO*1..10]->(n:Entity)
-        WHERE center.uuid = $center_uuid AND n.uuid IN [source.uuid, target.uuid]
-        RETURN min(length(p)) AS score, source.uuid AS source_uuid, target.uuid AS target_uuid
-        """,
-            edge_uuid=uuid,
-            center_uuid=center_node_uuid,
-        )
-        distance = 0.01
+    # Find the shortest path to center node
+    query = Query("""  
+        MATCH p = SHORTEST 1 (center:Entity {uuid: $center_uuid})-[:RELATES_TO]-+(n:Entity {uuid: $node_uuid})
+        RETURN length(p) AS score
+        """)
 
-        for record in records:
-            if (
-                record['source_uuid'] == center_node_uuid
-                or record['target_uuid'] == center_node_uuid
-            ):
-                continue
-            distance = record['score']
+    path_results = await asyncio.gather(
+        *[
+            driver.execute_query(
+                query,
+                node_uuid=uuid,
+                center_uuid=center_node_uuid,
+            )
+            for uuid in sorted_uuids
+        ]
+    )
 
-        if uuid in scores:
-            scores[uuid] = min(1 / distance, scores[uuid])
-        else:
-            scores[uuid] = 1 / distance
+    for uuid, result in zip(sorted_uuids, path_results):
+        records = result[0]
+        record = records[0] if len(records) > 0 else None
+        distance: float = record['score'] if record is not None else float('inf')
+        distance = 0 if uuid == center_node_uuid else distance
+        scores[uuid] = distance
 
     # rerank on shortest distance
-    sorted_uuids.sort(reverse=True, key=lambda cur_uuid: scores[cur_uuid])
+    sorted_uuids.sort(key=lambda cur_uuid: scores[cur_uuid])
 
     return sorted_uuids

graphiti_core/utils/maintenance/community_operations.py

@@ -0,0 +1,155 @@
+import asyncio
+import logging
+from collections import defaultdict
+from datetime import datetime
+
+from neo4j import AsyncDriver
+
+from graphiti_core.edges import CommunityEdge
+from graphiti_core.llm_client import LLMClient
+from graphiti_core.nodes import CommunityNode, EntityNode
+from graphiti_core.prompts import prompt_library
+from graphiti_core.utils.maintenance.edge_operations import build_community_edges
+
+logger = logging.getLogger(__name__)
+
+
+async def build_community_projection(driver: AsyncDriver) -> str:
+    records, _, _ = await driver.execute_query("""
+    CALL gds.graph.project("communities", "Entity",             
+        {RELATES_TO: {
+            type: "RELATES_TO",
+            orientation: "UNDIRECTED",
+            properties: {weight: {property: "*", aggregation: "COUNT"}}
+        }}
+    )
+    YIELD graphName AS graph, nodeProjection AS nodes, relationshipProjection AS edges
+    """)
+
+    return records[0]['graph']
+
+
+async def destroy_projection(driver: AsyncDriver, projection_name: str):
+    await driver.execute_query(
+        """
+    CALL gds.graph.drop($projection_name)
+    """,
+        projection_name=projection_name,
+    )
+
+
+async def get_community_clusters(
+    driver: AsyncDriver, projection_name: str
+) -> list[list[EntityNode]]:
+    records, _, _ = await driver.execute_query("""
+    CALL gds.leiden.stream("communities")
+    YIELD nodeId, communityId
+    RETURN gds.util.asNode(nodeId).uuid AS entity_uuid, communityId
+    """)
+    community_map: dict[int, list[str]] = defaultdict(list)
+    for record in records:
+        community_map[record['communityId']].append(record['entity_uuid'])
+
+    community_clusters: list[list[EntityNode]] = list(
+        await asyncio.gather(
+            *[EntityNode.get_by_uuids(driver, cluster) for cluster in community_map.values()]
+        )
+    )
+
+    return community_clusters
+
+
+async def summarize_pair(llm_client: LLMClient, summary_pair: tuple[str, str]) -> str:
+    # Prepare context for LLM
+    context = {'node_summaries': [{'summary': summary} for summary in summary_pair]}
+
+    llm_response = await llm_client.generate_response(
+        prompt_library.summarize_nodes.summarize_pair(context)
+    )
+
+    pair_summary = llm_response.get('summary', '')
+
+    return pair_summary
+
+
+async def generate_summary_description(llm_client: LLMClient, summary: str) -> str:
+    context = {'summary': summary}
+
+    llm_response = await llm_client.generate_response(
+        prompt_library.summarize_nodes.summary_description(context)
+    )
+
+    description = llm_response.get('description', '')
+
+    return description
+
+
+async def build_community(
+    llm_client: LLMClient, community_cluster: list[EntityNode]
+) -> tuple[CommunityNode, list[CommunityEdge]]:
+    summaries = [entity.summary for entity in community_cluster]
+    length = len(summaries)
+    while length > 1:
+        odd_one_out: str | None = None
+        if length % 2 == 1:
+            odd_one_out = summaries.pop()
+            length -= 1
+        new_summaries: list[str] = list(
+            await asyncio.gather(
+                *[
+                    summarize_pair(llm_client, (str(left_summary), str(right_summary)))
+                    for left_summary, right_summary in zip(
+                        summaries[: int(length / 2)], summaries[int(length / 2) :]
+                    )
+                ]
+            )
+        )
+        if odd_one_out is not None:
+            new_summaries.append(odd_one_out)
+        summaries = new_summaries
+        length = len(summaries)
+
+    summary = summaries[0]
+    name = await generate_summary_description(llm_client, summary)
+    now = datetime.now()
+    community_node = CommunityNode(
+        name=name,
+        group_id=community_cluster[0].group_id,
+        labels=['Community'],
+        created_at=now,
+        summary=summary,
+    )
+    community_edges = build_community_edges(community_cluster, community_node, now)
+
+    logger.info((community_node, community_edges))
+
+    return community_node, community_edges
+
+
+async def build_communities(
+    driver: AsyncDriver, llm_client: LLMClient
+) -> tuple[list[CommunityNode], list[CommunityEdge]]:
+    projection = await build_community_projection(driver)
+    community_clusters = await get_community_clusters(driver, projection)
+
+    communities: list[tuple[CommunityNode, list[CommunityEdge]]] = list(
+        await asyncio.gather(
+            *[build_community(llm_client, cluster) for cluster in community_clusters]
+        )
+    )
+
+    community_nodes: list[CommunityNode] = []
+    community_edges: list[CommunityEdge] = []
+    for community in communities:
+        community_nodes.append(community[0])
+        community_edges.extend(community[1])
+
+    await destroy_projection(driver, projection)
+    return community_nodes, community_edges
+
+
+async def remove_communities(driver: AsyncDriver):
+    await driver.execute_query("""
+    MATCH (c:Community)
+    DETACH DELETE c
+    """)