npcpy 1.1.28__py3-none-any.whl → 1.2.32__py3-none-any.whl

npcpy/memory/knowledge_graph.py

@@ -22,7 +22,8 @@ from npcpy.llm_funcs import (
     remove_idempotent_groups,
     zoom_in, 
     )
-from npcpy.npc_compiler import NPC
+
+from npcpy.memory.command_history import load_kg_from_db, save_kg_to_db
 
 def safe_kuzu_execute(conn, query, error_message="Kuzu query failed"):
     """Execute a Kuzu query with proper error handling"""
@@ -42,7 +43,7 @@ def create_group(conn, name: str, metadata: str = ""):
         return False
 
     try:
-        # Properly escape quotes in strings
+        
         escaped_name = name.replace('"', '\\"')
         escaped_metadata = metadata.replace('"', '\\"')
 
@@ -77,13 +78,13 @@ def init_db(db_path: str, drop=False):
         print("Database connection established successfully")
         
         if drop:
-            # Drop tables in reverse order of dependency
+            
             safe_kuzu_execute(conn, "DROP TABLE IF EXISTS Contains")
-            safe_kuzu_execute(conn, "DROP TABLE IF EXISTS EvolvedFrom") # New
+            safe_kuzu_execute(conn, "DROP TABLE IF EXISTS EvolvedFrom") 
             safe_kuzu_execute(conn, "DROP TABLE IF EXISTS Fact")
             safe_kuzu_execute(conn, "DROP TABLE IF EXISTS Groups")
 
-        # Fact table remains the same
+        
         safe_kuzu_execute(
             conn,
             """
@@ -97,7 +98,7 @@ def init_db(db_path: str, drop=False):
             "Failed to create Fact table",
         )
 
-        # UPDATED Groups table with generational properties
+        
         safe_kuzu_execute(
             conn,
             """
@@ -113,14 +114,14 @@ def init_db(db_path: str, drop=False):
         )
         print("Groups table (with generation tracking) created or already exists.")
         
-        # Contains relationship remains the same
+        
         safe_kuzu_execute(
             conn,
             "CREATE REL TABLE IF NOT EXISTS Contains(FROM Groups TO Fact);",
             "Failed to create Contains relationship table",
         )
         
-        # NEW EvolvedFrom relationship table
+        
         safe_kuzu_execute(
             conn,
             """
@@ -153,12 +154,12 @@ def find_similar_groups(
     **kwargs: Any
 ) -> List[str]:
     """Find existing groups that might contain this fact"""
-    response = conn.execute(f"MATCH (g:Groups) RETURN g.name;")  # Execute query
-    #print(response)
-    #print(type(response))
-    #print(dir(response))
+    response = conn.execute(f"MATCH (g:Groups) RETURN g.name;")  
+    
+    
+    
     groups = response.fetch_as_df()
-    #print(f"Groups: {groups}")
+    
     if not groups:
         return []
 
@@ -189,13 +190,14 @@ def find_similar_groups(
     return response["group_list"]
 
 
-def kg_initial(content_text=None,  
+def kg_initial(content,  
                model=None,
                provider=None,
                npc=None, 
                context='', 
                facts=None, 
-               generation=None):
+               generation=None, 
+               verbose=True,):
 
     if generation is None:
         CURRENT_GENERATION = 0
@@ -205,34 +207,99 @@ def kg_initial(content_text=None,
     print(f"--- Running KG Structuring Process (Generation: {CURRENT_GENERATION}) ---")
 
     if facts is None:
-        if not content_text:
+        if not content:
             raise ValueError("kg_initial requires either content_text or a list of facts.")
         print("  - Mode: Deriving new facts from text content...")
-        facts = get_facts(content_text, model=model, provider=provider, npc=npc, context=context)
-        for fact in facts:
+        all_facts = []
+        print(len(content))
+        if len(content)>10000:
+            # randomly sub sample 10000 characters
+            starting_point = random.randint(0, len(content)-10000)
+        
+            content_to_sample = content[starting_point:starting_point+10000]
+
+            for n in range(len(content)//10000):
+                print(n)
+                print(starting_point)
+                print(content_to_sample[0:1000])
+                facts = get_facts(content_to_sample,
+                                model=model, 
+                                provider=provider, 
+                                npc=npc, 
+                                context=context)
+                if verbose:
+                    print(f"    - Extracted {len(facts)} facts from segment {n+1}")
+                    print(facts)
+                all_facts.extend(facts)
+        else:
+            print(content[0:1000]   )
+            all_facts = get_facts(content, 
+                                  model=model, 
+                                  provider=provider, 
+                                  npc=npc, 
+                                  context=context)
+            if verbose:
+                print(f"    - Extracted {len(all_facts)} facts from content")
+                print(all_facts)            
+        for fact in all_facts:
+            
             fact['generation'] = CURRENT_GENERATION
     else:
         print(f"  - Mode: Building structure from {len(facts)} pre-existing facts...")
 
     print("  - Inferring implied facts (zooming in)...")
-    implied_facts = zoom_in(facts, model=model, provider=provider, npc=npc, context=context)
-    for fact in implied_facts:
+    all_implied_facts = []
+    if len(all_facts) > 20:
+        # sub sample facts randomly to generate zoomed in facts
+        sampled_facts = random.sample(all_facts, k=20)
+        for n in range(len(all_facts) // 20):
+            implied_facts = zoom_in(sampled_facts, 
+                                    model=model, 
+                                    provider=provider,
+                                npc=npc, 
+                                context=context)
+            all_implied_facts.extend(implied_facts)
+            if verbose:
+                print(f"    - Inferred {len(implied_facts)} implied facts from sample {n+1}")
+                print(implied_facts)
+    else:
+        implied_facts = zoom_in(all_facts, 
+                                model=model, 
+                                provider=provider,
+                                npc=npc, 
+                                context=context)
+        print(implied_facts)
+
+        all_implied_facts.extend(implied_facts)
+
+        if verbose:
+            print(f"    - Inferred {len(implied_facts)} implied facts from all facts")
+            print(implied_facts)
+    for fact in all_implied_facts:
         fact['generation'] = CURRENT_GENERATION
-    
-    all_facts = facts + implied_facts
-    
+
+    all_facts = all_facts + all_implied_facts
+
     print("  - Generating concepts from all facts...")
-    concepts = generate_groups(all_facts, model=model, provider=provider, npc=npc, context=context)
+    concepts = generate_groups(all_facts, 
+                               model=model, 
+                               provider=provider, 
+                               npc=npc, 
+                               context=context)
     for concept in concepts:
         concept['generation'] = CURRENT_GENERATION
-        
+    
+    if verbose:
+        print(f"    - Generated {len(concepts)} concepts")
+        print(concepts)
     print("  - Linking facts to concepts...")
     fact_to_concept_links = defaultdict(list)
     concept_names = [c['name'] for c in concepts if c and 'name' in c]
     for fact in all_facts:
 
         fact_to_concept_links[fact['statement']] = get_related_concepts_multi(fact['statement'], "fact", concept_names, model, provider, npc, context)
-        print(fact_to_concept_links[fact['statement']])
+        if verbose:
+            print(fact_to_concept_links[fact['statement']])
     print("  - Linking facts to other facts...")
     fact_to_fact_links = []
     fact_statements = [f['statement'] for f in all_facts]
@@ -249,7 +316,8 @@ def kg_initial(content_text=None,
             for related_stmt in related_fact_stmts:
 
                 fact_to_fact_links.append((fact['statement'], related_stmt))
-                print(fact['statement'], related_stmt)
+                if verbose:
+                    print(fact['statement'], related_stmt)
 
     return {
         "generation": CURRENT_GENERATION, 
@@ -261,9 +329,9 @@ def kg_initial(content_text=None,
     }
 
 
-
 def kg_evolve_incremental(existing_kg, 
-                          new_content_text, 
+                          new_content_text=None,
+                          new_facts=None, 
                           model = None, 
                           provider=None, 
                           npc=None, 
@@ -272,18 +340,16 @@ def kg_evolve_incremental(existing_kg,
                           link_concepts_facts = False, 
                           link_concepts_concepts=False, 
                           link_facts_facts = False, 
-                          
                           ):
 
     current_gen = existing_kg.get('generation', 0)
     next_gen = current_gen + 1
     print(f"\n--- ABSORBING INFO: Gen {current_gen} -> Gen {next_gen} ---")
 
-    print('extracting facts...')
-
     newly_added_concepts = []
     concept_links = list(existing_kg.get('concept_links', []))
-    fact_to_concept_links = defaultdict(list, existing_kg.get('fact_to_concept_links', {}))
+    fact_to_concept_links = defaultdict(list, 
+                                        existing_kg.get('fact_to_concept_links', {}))
     fact_to_fact_links = list(existing_kg.get('fact_to_fact_links', []))
 
     existing_facts = existing_kg.get('facts', [])
@@ -291,27 +357,51 @@ def kg_evolve_incremental(existing_kg,
     existing_concept_names = {c['name'] for c in existing_concepts}
     existing_fact_statements = [f['statement'] for f in existing_facts]
     all_concept_names = list(existing_concept_names)
+    
+    all_new_facts = []
+    print(npc, npc.model, npc.provider)
+    
+    if new_facts:
+        all_new_facts = new_facts
+        print(f'using pre-approved facts: {len(all_new_facts)}')
+    elif new_content_text:
+        print('extracting facts from content...')
+        if len(new_content_text) > 10000:
+            starting_point = random.randint(0, len(new_content_text)-10000)
+            for n in range(len(new_content_text)//10000):
+                content_to_sample = new_content_text[n*10000:(n+1)*10000]
+                facts = get_facts(content_to_sample, 
+                                model=model,
+                                provider=provider,
+                                npc = npc, 
+                                context=context)
+                all_new_facts.extend(facts)
+                print(facts)
+        else:
+            all_new_facts = get_facts(new_content_text, 
+                                model=model,
+                                provider=provider,
+                                npc = npc, 
+                                context=context)
+            print(all_new_facts)
+    else:
+        print("No new content or facts provided")
+        return existing_kg, {}
 
+    for fact in all_new_facts: 
+        fact['generation'] = next_gen
 
-    new_facts = get_facts(new_content_text, 
-                          model=model,
-                          provider=provider,
-                          npc = npc, 
-                          context=context)
+    final_facts = existing_facts + all_new_facts
 
-    for fact in new_facts: 
-        fact['generation'] = next_gen
-    
-    final_facts = existing_facts + new_facts
-    
     if get_concepts:
         print('generating groups...')
 
-        candidate_concepts = generate_groups(new_facts, 
+        candidate_concepts = generate_groups(all_new_facts, 
                                             model = model, 
                                             provider = provider, 
                                             npc=npc, 
                                             context=context)
+        print(candidate_concepts)
         print('checking group uniqueness')
         for cand_concept in candidate_concepts:
             cand_name = cand_concept['name']
@@ -331,7 +421,6 @@ def kg_evolve_incremental(existing_kg,
                                                             context)
                 for related_name in related_concepts:
                     if related_name != cand_name: 
-                        # denying self 
                         concept_links.append((cand_name, related_name))
             all_concept_names.append(cand_name)
 
@@ -339,21 +428,26 @@ def kg_evolve_incremental(existing_kg,
 
         if link_concepts_facts:
             print('linking facts and concepts...')
-            for fact in new_facts:
+            for fact in all_new_facts:
                 fact_to_concept_links[fact['statement']] = get_related_concepts_multi(fact['statement'], 
                                                                                     "fact", 
                                                                                     all_concept_names, 
-                                                                                    model, 
-                                                                                    provider,
-                                                                                    npc,  
-                                                                                    context)
+                                                                                    model = model,
+                                                                                    provider=provider,
+                                                                                    npc = npc, 
+                                                                                    context= context)
     else:
         final_concepts = existing_concepts
     if link_facts_facts:
         print('linking facts and facts...')
 
-        for new_fact in new_facts:
-            related_fact_stmts = get_related_facts_llm(new_fact['statement'], existing_fact_statements, model, provider, context)
+        for new_fact in all_new_facts:
+            related_fact_stmts = get_related_facts_llm(new_fact['statement'], 
+                                                       existing_fact_statements, 
+                                                       model = model,
+                                                       provider = provider,
+                                                       npc = npc, 
+                                                       context=context)
             for related_stmt in related_fact_stmts:
                 fact_to_fact_links.append((new_fact['statement'], related_stmt))
                 
@@ -370,20 +464,25 @@ def kg_evolve_incremental(existing_kg,
 
 
 
-# UPGRADED KG_SLEEP_PROCESS - Now correctly uses the kg_initial framework.
-def kg_sleep_process(existing_kg, model=None, provider=None, npc=None, context='', operations_config=None):
+
+def kg_sleep_process(existing_kg, 
+                     model=None, 
+                     provider=None, 
+                     npc=None, 
+                     context='', 
+                     operations_config=None):
     current_gen = existing_kg.get('generation', 0)
     next_gen = current_gen + 1
     print(f"\n--- SLEEPING (Evolving Knowledge): Gen {current_gen} -> Gen {next_gen} ---")
 
-    # Load KG components into mutable structures
+    
     facts_map = {f['statement']: f for f in existing_kg.get('facts', [])}
     concepts_map = {c['name']: c for c in existing_kg.get('concepts', [])}
     fact_links = defaultdict(list, {k: list(v) for k, v in existing_kg.get('fact_to_concept_links', {}).items()})
     concept_links = set(tuple(sorted(link)) for link in existing_kg.get('concept_links', []))
     fact_to_fact_links = set(tuple(sorted(link)) for link in existing_kg.get('fact_to_fact_links', []))
 
-    # --- PHASE 1: BOOTSTRAP using kg_initial ---
+    
     print("  - Phase 1: Checking for unstructured facts...")
     facts_with_concepts = set(fact_links.keys())
     orphaned_fact_statements = list(set(facts_map.keys()) - facts_with_concepts)
@@ -392,7 +491,7 @@ def kg_sleep_process(existing_kg, model=None, provider=None, npc=None, context='
         print(f"    - Found {len(orphaned_fact_statements)} orphaned facts. Applying full KG structuring process...")
         orphaned_facts_as_dicts = [facts_map[s] for s in orphaned_fact_statements]
         
-        # USE THE REFACTORED KG_INITIAL AS THE STRUCTURING ENGINE
+        
         new_structure = kg_initial(
             facts=orphaned_facts_as_dicts,
             model=model,
@@ -402,7 +501,7 @@ def kg_sleep_process(existing_kg, model=None, provider=None, npc=None, context='
             generation=next_gen
         )
 
-        # Merge the newly generated structure back into the main KG
+        
         print("    - Merging new structure into main KG...")
         for concept in new_structure.get("concepts", []):
             if concept['name'] not in concepts_map:
@@ -418,7 +517,7 @@ def kg_sleep_process(existing_kg, model=None, provider=None, npc=None, context='
     else:
         print("    - Knowledge graph is sufficiently structured. Proceeding to refinement.")
 
-    # --- PHASE 2: REFINE ---
+    
     if operations_config is None:
         possible_ops = ['prune', 'deepen', 'abstract_link']
         ops_to_run = random.sample(possible_ops, k=random.randint(1, 2))
@@ -428,7 +527,7 @@ def kg_sleep_process(existing_kg, model=None, provider=None, npc=None, context='
     print(f"  - Phase 2: Executing refinement operations: {ops_to_run}")
 
     for op in ops_to_run:
-        # Prune Operation
+        
         if op == 'prune' and (len(facts_map) > 10 or len(concepts_map) > 5):
             print("    - Running 'prune' operation using consolidate_facts_llm...")
             fact_to_check = random.choice(list(facts_map.values()))
@@ -438,7 +537,7 @@ def kg_sleep_process(existing_kg, model=None, provider=None, npc=None, context='
                 print(f"      - Pruning redundant fact: '{fact_to_check['statement'][:80]}...'")
                 del facts_map[fact_to_check['statement']]
 
-        # Deepen Operation
+        
         elif op == 'deepen' and facts_map:
             print("    - Running 'deepen' operation using zoom_in...")
             fact_to_deepen = random.choice(list(facts_map.values()))
@@ -454,7 +553,7 @@ def kg_sleep_process(existing_kg, model=None, provider=None, npc=None, context='
         else:
             print(f"    - SKIPPED: Operation '{op}' did not run (conditions not met).")
         
-    # Reassemble the final KG
+    
     new_kg = {
         "generation": next_gen, 
         "facts": list(facts_map.values()), 
@@ -545,15 +644,15 @@ def store_fact_and_group(conn, fact: str,
         return False
     
     print(f"store_fact_and_group: Storing fact: {fact}, with groups:"
-          f" {groups}") # DEBUG
+          f" {groups}") 
     try:
-        # Insert the fact
-        insert_success = insert_fact(conn, fact, path) # Capture return
+        
+        insert_success = insert_fact(conn, fact, path) 
         if not insert_success:
             print(f"store_fact_and_group: Failed to insert fact: {fact}")
             return False
         
-        # Assign fact to groups
+        
         for group in groups:
             assign_success = assign_fact_to_group_graph(conn, fact, group)
             if not assign_success:
@@ -573,19 +672,19 @@ def insert_fact(conn, fact: str, path: str) -> bool:
               " database connection is None")
         return False
     try:
-        # Properly escape quotes in strings
+        
         escaped_fact = fact.replace('"', '\\"')
         escaped_path = os.path.expanduser(path).replace('"', '\\"')
 
-        # Generate timestamp
+        
         timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
-        print(f"insert_fact: Attempting to insert fact: {fact}") #DEBUG
+        print(f"insert_fact: Attempting to insert fact: {fact}") 
 
-        # Begin transaction
+        
         safe_kuzu_execute(conn, "BEGIN TRANSACTION")
 
-        # Check if fact already exists
+        
         check_query = f"""
         MATCH (f:Fact {{content: "{escaped_fact}"}})
         RETURN f
@@ -599,7 +698,7 @@ def insert_fact(conn, fact: str, path: str) -> bool:
             print(f"insert_fact: Error checking if fact exists: {error}")
             return False
 
-        # Insert fact if it doesn't exist
+        
         if not result.has_next():
             insert_query = f"""
             CREATE (f:Fact {{
@@ -617,7 +716,7 @@ def insert_fact(conn, fact: str, path: str) -> bool:
                 print(f"insert_fact: Error inserting fact: {error}")
                 return False
 
-        # Commit transaction
+        
         safe_kuzu_execute(conn, "COMMIT")
         print(f"insert_fact: Successfully inserted/found fact: {fact}")
         return True
@@ -636,14 +735,14 @@ def assign_fact_to_group_graph(conn, fact: str, group: str) -> bool:
         return False
 
     try:
-        # Properly escape quotes in strings
+        
         escaped_fact = fact.replace('"', '\\"')
         escaped_group = group.replace('"', '\\"')
 
         print(f"assign_fact_to_group_graph: Assigning fact: {fact} to group:"
-              f" {group}") #DEBUG
+              f" {group}") 
 
-        # Check if both fact and group exist before creating relationship
+        
         check_query = f"""
         MATCH (f:Fact {{content: "{escaped_fact}"}})
         RETURN f
@@ -670,7 +769,7 @@ def assign_fact_to_group_graph(conn, fact: str, group: str) -> bool:
             print(f"assign_fact_to_group_graph: Group not found: {group}")
             return False
 
-        # Create relationship
+        
         query = f"""
         MATCH (f:Fact), (g:Groups)
         WHERE f.content = "{escaped_fact}" AND g.name = "{escaped_group}"
@@ -695,26 +794,26 @@ def assign_fact_to_group_graph(conn, fact: str, group: str) -> bool:
         traceback.print_exc()
         return False
 
-#--- Kuzu Database integration ---
+
 def store_fact_and_group(conn, fact: str, groups: List[str], path: str) -> bool:
     """Insert a fact into the database along with its groups"""
     if not conn:
         print("store_fact_and_group: Database connection is None")
         return False
     
-    print(f"store_fact_and_group: Storing fact: {fact}, with groups: {groups}") # DEBUG
+    print(f"store_fact_and_group: Storing fact: {fact}, with groups: {groups}") 
     try:
-        # Insert the fact
-        insert_success = insert_fact(conn, fact, path) # Capture return value
+        
+        insert_success = insert_fact(conn, fact, path) 
         if not insert_success:
-            print(f"store_fact_and_group: Failed to insert fact: {fact}") #DEBUG
+            print(f"store_fact_and_group: Failed to insert fact: {fact}") 
             return False
         
-        # Assign fact to groups
+        
         for group in groups:
             assign_success = assign_fact_to_group_graph(conn, fact, group)
             if not assign_success:
-                print(f"store_fact_and_group: Failed to assign fact {fact} to group {group}") #DEBUG
+                print(f"store_fact_and_group: Failed to assign fact {fact} to group {group}") 
                 return False
         
         return True
@@ -724,7 +823,7 @@ def store_fact_and_group(conn, fact: str, groups: List[str], path: str) -> bool:
         return False
     
         
-# ---Database and other helper methods---
+
 def safe_kuzu_execute(conn, query, error_message="Kuzu query failed"):
     """Execute a Kuzu query with proper error handling"""
     try:
@@ -763,7 +862,7 @@ def process_text_with_chroma(
     Returns:
         List of extracted facts
     """
-    # Initialize databases
+    
     kuzu_conn = init_db(kuzu_db_path, drop=False)
     chroma_client, chroma_collection = setup_chroma_db( 
         "knowledge_graph",
@@ -771,15 +870,15 @@ def process_text_with_chroma(
         chroma_db_path
     )
 
-    # Extract facts
+    
     facts = extract_facts(text, model=model, provider=provider, npc=npc)
 
-    # Process extracted facts
+    
     for i in range(0, len(facts), batch_size):
         batch = facts[i : i + batch_size]
         print(f"\nProcessing batch {i//batch_size + 1} ({len(batch)} facts)")
 
-        # Generate embeddings for the batch using npcpy.llm_funcs.get_embeddings
+        
         from npcpy.llm_funcs import get_embeddings
 
         batch_embeddings = get_embeddings(
@@ -790,7 +889,7 @@ def process_text_with_chroma(
             print(f"Processing fact: {fact}")
             embedding = batch_embeddings[j]
 
-            # Check for similar facts in Chroma before inserting
+            
             similar_facts = find_similar_facts_chroma(
                 chroma_collection, fact, query_embedding=embedding, n_results=3
             )
@@ -799,9 +898,9 @@ def process_text_with_chroma(
                 print(f"Similar facts found:")
                 for result in similar_facts:
                     print(f"  - {result['fact']} (distance: {result['distance']})")
-                # Note: Could implement a similarity threshold here to skip highly similar facts
+                
 
-            # Prepare metadata
+            
             metadata = {
                 "path": path,
                 "timestamp": datetime.now().isoformat(),
@@ -809,10 +908,10 @@ def process_text_with_chroma(
                 "source_provider": provider,
             }
 
-            # Insert into Kuzu graph DB
+            
             kuzu_success = insert_fact(kuzu_conn, fact, path)
 
-            # Insert into Chroma vector DB if Kuzu insert was successful
+            
             if kuzu_success:
                 chroma_id = store_fact_with_embedding(
                     chroma_collection, fact, metadata, embedding
@@ -824,7 +923,7 @@ def process_text_with_chroma(
             else:
                 print(f"Failed to save fact to Kuzu graph")
 
-    # Close Kuzu connection
+    
     kuzu_conn.close()
 
     return facts
@@ -853,12 +952,12 @@ def hybrid_search_with_chroma(
     Returns:
         List of dictionaries with combined results
     """
-    # Get embedding for query using npcpy.llm_funcs.get_embeddings
+    
     from npcpy.llm_funcs import get_embeddings
 
     query_embedding = get_embeddings([query])[0]
 
-    # Step 1: Find similar facts using Chroma vector search
+    
     vector_results = find_similar_facts_chroma(
         chroma_collection,
         query,
@@ -867,13 +966,13 @@ def hybrid_search_with_chroma(
         metadata_filter=metadata_filter,
     )
 
-    # Extract just the fact texts from vector results
+    
     vector_facts = [result["fact"] for result in vector_results]
 
-    # Step 2: Expand context using graph relationships
+    
     expanded_results = []
 
-    # Add vector search results
+    
     for result in vector_results:
         expanded_results.append(
             {
@@ -885,13 +984,13 @@ def hybrid_search_with_chroma(
             }
         )
 
-    # For each vector-matched fact, find related facts in the graph
+    
     for fact in vector_facts:
         try:
-            # Safely escape fact text for Kuzu query
+            
             escaped_fact = fact.replace('"', '\\"')
 
-            # Find groups containing this fact
+            
             group_result = kuzu_conn.execute(
                 f"""
                 MATCH (g:Groups)-[:Contains]->(f:Fact)
@@ -900,18 +999,18 @@ def hybrid_search_with_chroma(
                 """
             ).get_as_df()
 
-            # Extract group names
+            
             fact_groups = [row["g.name"] for _, row in group_result.iterrows()]
 
-            # Apply group filter if provided
+            
             if group_filter:
                 fact_groups = [g for g in fact_groups if g in group_filter]
 
-            # For each group, find other related facts
+            
             for group in fact_groups:
                 escaped_group = group.replace('"', '\\"')
 
-                # Find facts in the same group
+                
                 related_facts_result = kuzu_conn.execute(
                     f"""
                     MATCH (g:Groups)-[:Contains]->(f:Fact)
@@ -921,7 +1020,7 @@ def hybrid_search_with_chroma(
                     """
                 ).get_as_df()
 
-                # Add these related facts to results
+                
                 for _, row in related_facts_result.iterrows():
                     related_fact = {
                         "fact": row["f.content"],
@@ -931,7 +1030,7 @@ def hybrid_search_with_chroma(
                         "recorded_at": row["f.recorded_at"],
                     }
 
-                    # Avoid duplicates
+                    
                     if not any(
                         r.get("fact") == related_fact["fact"] for r in expanded_results
                     ):
@@ -940,7 +1039,7 @@ def hybrid_search_with_chroma(
         except Exception as e:
             print(f"Error expanding results via graph: {e}")
 
-    # Return results, limiting to top_k if needed
+    
     return expanded_results[:top_k]
 
 
@@ -964,14 +1063,14 @@ def find_similar_facts_chroma(
         List of dictionaries with results
     """
     try:
-        # Perform query with optional metadata filtering
+        
         results = collection.query(
             query_embeddings=[query_embedding],
             n_results=n_results,
             where=metadata_filter,
         )
 
-        # Format results
+        
         formatted_results = []
         for i, doc in enumerate(results["documents"][0]):
             formatted_results.append(
@@ -1007,12 +1106,12 @@ def store_fact_with_embedding(
         ID of the stored fact
     """
     try:
-        # Generate a deterministic ID from the fact content
+        
         import hashlib
 
         fact_id = hashlib.md5(fact.encode()).hexdigest()
 
-        # Store document with pre-generated embedding
+        
         collection.add(
             documents=[fact],
             embeddings=[embedding],
@@ -1033,7 +1132,7 @@ def save_facts_to_graph_db(
         batch = facts[i : i + batch_size]
         print(f"\nProcessing batch {i//batch_size + 1} ({len(batch)} facts)")
 
-        # Process each fact in the batch
+        
         for fact in batch:
             try:
                 print(f"Inserting fact: {fact}")
@@ -1049,3 +1148,271 @@ def save_facts_to_graph_db(
                 continue
 
         print(f"Completed batch {i//batch_size + 1}")
+
+
+
+def kg_add_fact(
+   engine, 
+   fact_text: str, 
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """Add a new fact to the knowledge graph"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   new_fact = {
+       "statement": fact_text,
+       "source_text": fact_text,
+       "type": "manual",
+       "generation": kg_data.get('generation', 0),
+       "origin": "manual_add"
+   }
+   
+   kg_data['facts'].append(new_fact)
+   save_kg_to_db(engine, kg_data, team_name, npc_name, directory_path)
+   
+   return f"Added fact: {fact_text}"
+
+def kg_search_facts(
+   engine,
+   query: str, 
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """Search facts in the knowledge graph"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   matching_facts = []
+   for fact in kg_data.get('facts', []):
+       if query.lower() in fact['statement'].lower():
+           matching_facts.append(fact['statement'])
+   
+   return matching_facts
+
+def kg_remove_fact(
+   engine,
+   fact_text: str, 
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """Remove a fact from the knowledge graph"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   original_count = len(kg_data.get('facts', []))
+   kg_data['facts'] = [f for f in kg_data.get('facts', []) if f['statement'] != fact_text]
+   removed_count = original_count - len(kg_data['facts'])
+   
+   if removed_count > 0:
+       save_kg_to_db(engine, kg_data, team_name, npc_name, directory_path)
+       return f"Removed {removed_count} matching fact(s)"
+   
+   return "No matching facts found"
+
+def kg_list_concepts(
+   engine,
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """List all concepts in the knowledge graph"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   concepts = [c['name'] for c in kg_data.get('concepts', [])]
+   return concepts
+
+def kg_get_facts_for_concept(
+   engine,
+   concept_name: str, 
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """Get all facts linked to a specific concept"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   fact_to_concept_links = kg_data.get('fact_to_concept_links', {})
+   linked_facts = []
+   
+   for fact_statement, linked_concepts in fact_to_concept_links.items():
+       if concept_name in linked_concepts:
+           linked_facts.append(fact_statement)
+   
+   return linked_facts
+
+def kg_add_concept(
+   engine,
+   concept_name: str, 
+   concept_description: str,
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """Add a new concept to the knowledge graph"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   new_concept = {
+       "name": concept_name,
+       "description": concept_description,
+       "generation": kg_data.get('generation', 0)
+   }
+   
+   kg_data['concepts'].append(new_concept)
+   save_kg_to_db(engine, kg_data, team_name, npc_name, directory_path)
+   
+   return f"Added concept: {concept_name}"
+
+def kg_remove_concept(
+   engine,
+   concept_name: str, 
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """Remove a concept from the knowledge graph"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   original_count = len(kg_data.get('concepts', []))
+   kg_data['concepts'] = [c for c in kg_data.get('concepts', []) if c['name'] != concept_name]
+   removed_count = original_count - len(kg_data['concepts'])
+   
+   if removed_count > 0:
+       save_kg_to_db(engine, kg_data, team_name, npc_name, directory_path)
+       return f"Removed concept: {concept_name}"
+   
+   return "Concept not found"
+
+def kg_link_fact_to_concept(
+   engine,
+   fact_text: str,
+   concept_name: str, 
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """Link a fact to a concept in the knowledge graph"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   fact_to_concept_links = kg_data.get('fact_to_concept_links', {})
+   
+   if fact_text not in fact_to_concept_links:
+       fact_to_concept_links[fact_text] = []
+   
+   if concept_name not in fact_to_concept_links[fact_text]:
+       fact_to_concept_links[fact_text].append(concept_name)
+       kg_data['fact_to_concept_links'] = fact_to_concept_links
+       save_kg_to_db(engine, kg_data, team_name, npc_name, directory_path)
+       return f"Linked fact '{fact_text}' to concept '{concept_name}'"
+   
+   return "Fact already linked to concept"
+
+def kg_get_all_facts(
+   engine,
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """Get all facts from the knowledge graph"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   facts = [f['statement'] for f in kg_data.get('facts', [])]
+   return facts
+
+def kg_get_stats(
+   engine,
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """Get statistics about the knowledge graph"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   return {
+       "total_facts": len(kg_data.get('facts', [])),
+       "total_concepts": len(kg_data.get('concepts', [])),
+       "total_fact_concept_links": len(kg_data.get('fact_to_concept_links', {})),
+       "generation": kg_data.get('generation', 0)
+   }
+
+def kg_evolve_knowledge(
+   engine,
+   content_text: str,
+   npc=None, 
+   team=None,
+   model=None,
+   provider=None
+):
+   """Evolve the knowledge graph with new content"""
+   directory_path = os.getcwd()
+   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
+   npc_name = npc.name if npc else 'default_npc'
+   
+   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)
+   
+   evolved_kg, _ = kg_evolve_incremental(
+       existing_kg=kg_data,
+       new_content_text=content_text,
+       model=npc.model if npc else model,
+       provider=npc.provider if npc else provider,
+       npc=npc,
+       get_concepts=True,
+       link_concepts_facts=False,
+       link_concepts_concepts=False,
+       link_facts_facts=False
+   )
+   
+   save_kg_to_db(engine, evolved_kg, team_name, npc_name, directory_path)
+   
+   return "Knowledge graph evolved with new content"