kgnode 0.1.0__py3-none-any.whl

kgnode/__init__.py

@@ -0,0 +1,60 @@
+"""
+kgnode - Knowledge Graph Agnostic Node for Knowledge-Aware LLM Applications.
+
+Public API for knowledge graph retrieval and answer generation.
+"""
+
+# Main Pipeline APIs
+from kgnode.seed_finder import citable, get_seed_nodes
+from kgnode.subgraph_extraction import get_subgraphs
+from kgnode.generator import (
+    generate_sparql,
+    kg_retrieve,
+    generate_answer,
+    generate_answer_using_subgraph,
+)
+
+# Validation
+from kgnode.validator import validate_subgraph
+
+# Search Operations
+from kgnode.keyword_search import search_entities_by_keywords
+
+# VectorDB Operations
+from kgnode.chroma_db import (
+    compile_chromadb,
+    compile_chromadb_from_csv,
+    semantic_search_entities,
+    get_or_create_chromadb,
+    add_or_update_entities,
+    delete_entities,
+)
+
+# Core Configuration
+from kgnode.core.kg_config import KGConfig
+from kgnode.core.sparql_query import execute_sparql_query
+
+__all__ = [
+    # Main Pipeline APIs
+    "citable",
+    "get_seed_nodes",
+    "get_subgraphs",
+    "generate_sparql",
+    "kg_retrieve",
+    "generate_answer",
+    "generate_answer_using_subgraph",
+    # Validation
+    "validate_subgraph",
+    # Search Operations
+    "search_entities_by_keywords",
+    # VectorDB Operations
+    "compile_chromadb",
+    "compile_chromadb_from_csv",
+    "semantic_search_entities",
+    "get_or_create_chromadb",
+    "add_or_update_entities",
+    "delete_entities",
+    # Core Configuration
+    "KGConfig",
+    "execute_sparql_query",
+]

kgnode/_entity_descriptor.py

@@ -0,0 +1,474 @@
+from kgnode.core.sparql_query import execute_sparql_query
+from typing import List, Dict, Callable, Optional, Any
+from kgnode.core.kg_config import KGConfig
+
+
+def _default_entity_descriptor_logic(entity_uri: str, triples: List[Dict[str, Any]]) -> str:
+    """Default DBLP-specific entity descriptor logic.
+
+    This function contains the hardcoded DBLP logic for creating entity descriptions.
+    It can be replaced by user-provided functions for other knowledge graphs.
+
+    Args:
+        entity_uri: URI of the entity (cleaned, no brackets).
+        triples: List of dicts with 'predicate' and 'object' keys.
+
+    Returns:
+        Natural language description optimized for search.
+    """
+    if not triples:
+        return _uri_to_label(entity_uri)
+
+    # Organize triples by priority
+    entity_type = None
+    title = None
+    authors = []
+    venue = None
+    year = None
+    affiliation = None
+    coauthors = []
+
+    for triple in triples:
+        predicate = triple['predicate']
+        obj = triple['object']
+        pred_label = _uri_to_label(predicate).lower()
+        obj_label = _uri_to_label(obj)
+
+        # Identify entity type
+        if 'type' in pred_label:
+            entity_type = obj_label.lower()
+
+        # Extract key fields based on predicate
+        elif 'title' in pred_label:
+            title = obj_label
+        elif 'authored by' in pred_label or 'author' in pred_label or 'creator' in pred_label:
+            authors.append(obj_label)
+        elif 'published in' in pred_label or 'venue' in pred_label or 'journal' in pred_label:
+            venue = obj_label
+        elif 'year' in pred_label:
+            year = obj_label
+        elif 'affiliation' in pred_label or 'organization' in pred_label:
+            affiliation = obj_label
+        elif 'coauthor' in pred_label or 'collaborate' in pred_label:
+            coauthors.append(obj_label)
+
+    # Get base entity label
+    entity_label = _uri_to_label(entity_uri)
+
+    # Build focused description based on entity type
+    description_parts = []
+
+    # Always start with the entity label/name
+    if entity_type == 'person' or entity_type == 'creator':
+        description_parts.append(f"Person: {entity_label}")
+        if affiliation:
+            description_parts.append(f"affiliated with {affiliation}")
+        if authors:  # These are actually papers they authored
+            description_parts.append(f"author of {len(authors)} publications")
+        if coauthors:
+            coauthor_names = ", ".join(coauthors[:5])
+            description_parts.append(f"collaborates with {coauthor_names}")
+
+    elif entity_type in ['article', 'publication', 'inproceedings', 'informal']:
+        description_parts.append(f"Publication: {entity_label}")
+        if title:
+            description_parts.append(f"titled '{title}'")
+        if authors:
+            # Limit to first few authors for readability
+            author_names = ", ".join(authors[:10])
+            if len(authors) > 10:
+                author_names += f" and {len(authors) - 10} more"
+            description_parts.append(f"authored by {author_names}")
+        if venue:
+            description_parts.append(f"published in {venue}")
+        if year:
+            description_parts.append(f"in year {year}")
+
+    else:
+        # Generic fallback for other entity types
+        description_parts.append(f"{entity_type or 'Entity'}: {entity_label}")
+        if title:
+            description_parts.append(f"titled '{title}'")
+        if authors:
+            description_parts.append(f"associated with authors: {', '.join(authors[:5])}")
+        if venue:
+            description_parts.append(f"venue: {venue}")
+        if year:
+            description_parts.append(f"year: {year}")
+
+    # Join with proper punctuation
+    return ". ".join(description_parts) + "."
+
+
+def _default_relation_descriptor_logic(relation_uri: str) -> str:
+    """Default relation descriptor logic (URI to label conversion).
+
+    Args:
+        relation_uri: URI of the relation/predicate.
+
+    Returns:
+        Human-readable label.
+    """
+    return _uri_to_label(relation_uri)
+
+
+def create_entity_description(entity_uri: str, config: Optional[KGConfig] = None) -> str:
+    """
+    Create a focused natural language description optimized for semantic search.
+    Prioritizes key identifying information: names, titles, venues, years.
+
+    Note: This function uses default DBLP logic. For custom logic, use EntityDescriptorWrapper
+    with KGConfig.
+
+    Args:
+        entity_uri: URI of the entity (without angle brackets)
+        config: Optional KGConfig instance for configuration.
+            If None, uses default KGConfig with environment variables or built-in defaults.
+
+    Returns:
+        Natural language description optimized for search
+    """
+    # Initialize config if not provided
+    if config is None:
+        config = KGConfig.default()
+
+    # Remove angle brackets if present
+    entity_uri = entity_uri.strip()
+    if entity_uri.startswith('<') and entity_uri.endswith('>'):
+        entity_uri = entity_uri[1:-1]
+
+    query = f"""
+    PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    PREFIX dblp: <https://dblp.org/rdf/schema#>
+    SELECT ?predicate ?object
+    WHERE {{
+        <{entity_uri}> ?predicate ?object .
+    }}
+    """
+
+    triples = execute_sparql_query(query, config=config)
+
+    return _default_entity_descriptor_logic(entity_uri, triples)
+
+def create_entity_descriptions_batch(entity_uris: List[str], config: Optional[KGConfig] = None) -> Dict[str, str]:
+    """
+    Create focused descriptions for multiple entities in batch.
+    Optimized for semantic search of author names and paper titles.
+
+    Args:
+        entity_uris: List of entity URIs to describe.
+        config: Optional KGConfig instance for configuration.
+            If None, uses default KGConfig with environment variables or built-in defaults.
+
+    Returns:
+        Dictionary mapping entity URIs to their descriptions.
+    """
+    # Initialize config if not provided
+    if config is None:
+        config = KGConfig.default()
+
+    if not entity_uris:
+        return {}
+
+    # Clean URIs
+    cleaned_uris = []
+    for uri in entity_uris:
+        uri = uri.strip()
+        if not uri:
+            continue
+        if uri.startswith('<') and uri.endswith('>'):
+            uri = uri[1:-1]
+        cleaned_uris.append(uri)
+
+    if not cleaned_uris:
+        return {}
+
+    # Build VALUES clause
+    values_clause = " ".join([f"<{uri}>" for uri in cleaned_uris])
+
+    query = f"""
+    PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    PREFIX dblp: <https://dblp.org/rdf/schema#>
+
+    SELECT ?entity ?predicate ?object
+    WHERE {{
+        VALUES ?entity {{ {values_clause} }}
+        ?entity ?predicate ?object .
+    }}
+    """
+
+    triples = execute_sparql_query(query, config=config)
+
+    # Group triples by entity
+    entity_triples = {}
+    for triple in triples:
+        entity = triple['entity']
+        if entity not in entity_triples:
+            entity_triples[entity] = []
+        entity_triples[entity].append(triple)
+
+    # Create descriptions using the same logic
+    descriptions = {}
+    for original_uri, cleaned_uri in zip(entity_uris, cleaned_uris):
+        if cleaned_uri not in entity_triples:
+            descriptions[original_uri] = _uri_to_label(original_uri)
+        else:
+            # Use similar logic as create_entity_description
+            entity_type = None
+            title = None
+            authors = []
+            venue = None
+            year = None
+            affiliation = None
+
+            for triple in entity_triples[cleaned_uri]:
+                pred_label = _uri_to_label(triple['predicate']).lower()
+                obj_label = _uri_to_label(triple['object'])
+
+                if 'type' in pred_label:
+                    entity_type = obj_label.lower()
+                elif 'title' in pred_label:
+                    title = obj_label
+                elif 'authored by' in pred_label or 'author' in pred_label or 'creator' in pred_label:
+                    authors.append(obj_label)
+                elif 'published in' in pred_label or 'venue' in pred_label or 'journal' in pred_label:
+                    venue = obj_label
+                elif 'year' in pred_label:
+                    year = obj_label
+                elif 'affiliation' in pred_label:
+                    affiliation = obj_label
+
+            entity_label = _uri_to_label(cleaned_uri)
+            description_parts = []
+
+            if entity_type == 'person' or entity_type == 'creator':
+                description_parts.append(f"Person: {entity_label}")
+                if affiliation:
+                    description_parts.append(f"affiliated with {affiliation}")
+                if len(authors) > 0:
+                    description_parts.append(f"author of {len(authors)} publications")
+
+            elif entity_type in ['article', 'publication', 'inproceedings', 'informal']:
+                description_parts.append(f"Publication: {entity_label}")
+                if title:
+                    description_parts.append(f"titled '{title}'")
+                if authors:
+                    author_names = ", ".join(authors[:10])
+                    if len(authors) > 10:
+                        author_names += f" and {len(authors) - 10} more"
+                    description_parts.append(f"authored by {author_names}")
+                if venue:
+                    description_parts.append(f"published in {venue}")
+                if year:
+                    description_parts.append(f"in year {year}")
+            else:
+                description_parts.append(f"{entity_type or 'Entity'}: {entity_label}")
+
+            descriptions[original_uri] = ". ".join(description_parts) + "." if description_parts else entity_label
+
+    return descriptions
+
+def create_relation_description(relation_uri: str) -> str:
+    """
+    Create natural language description for a relation.
+    Knowledge graph agnostic - converts URI to readable text.
+
+    Args:
+        relation_uri: URI of the relation/predicate
+
+    Returns:
+        Natural language description
+    """
+    return _uri_to_label(relation_uri)
+
+def _uri_to_label(uri: str) -> str:
+    """
+    Convert URI to human-readable label.
+    Works for any knowledge graph by extracting and formatting the URI fragment.
+
+    Args:
+        uri: Full URI (e.g., "http://example.org/ontology#hasName" or "<http://example.org/ontology#hasName>")
+
+    Returns:
+        Human-readable label (e.g., "has name")
+    """
+    # Remove SPARQL brackets if present
+    uri = uri.strip('<>')
+
+    # Extract the fragment/local name from URI
+    if '#' in uri:
+        label = uri.split('#')[-1]
+    elif '/' in uri:
+        label = uri.split('/')[-1]
+    else:
+        label = uri
+
+    # Convert camelCase or PascalCase to spaces
+    # hasName -> has Name -> has name
+    import re
+    label = re.sub(r'([a-z])([A-Z])', r'\1 \2', label)
+
+    # Convert snake_case or kebab-case to spaces
+    label = label.replace('_', ' ').replace('-', ' ')
+
+    # Lowercase and clean up
+    label = label.lower().strip()
+
+    return label
+
+
+class EntityDescriptorWrapper:
+    """Wrapper class that handles SPARQL queries and applies descriptor logic.
+
+    This class provides both single and batch entity description functionality,
+    using either user-provided descriptor logic or default DBLP logic.
+    """
+
+    def __init__(
+        self,
+        descriptor_function: Optional[Callable[[str, List[Dict[str, Any]]], str]] = None,
+        config: Optional[KGConfig] = None
+    ):
+        """Initialize the wrapper with descriptor function.
+
+        Args:
+            descriptor_function: Function that takes (entity_uri, triples) and returns
+                description string. If None, uses default DBLP logic.
+            config: Optional KGConfig instance for configuration.
+                If None, uses default KGConfig with environment variables or built-in defaults.
+        """
+        # Initialize config if not provided
+        if config is None:
+            from kgnode.core.kg_config import KGConfig
+            config = KGConfig.default()
+
+        self.descriptor_function = descriptor_function or _default_entity_descriptor_logic
+        self.config = config
+
+    def describe_single(self, entity_uri: str) -> str:
+        """Create description for a single entity.
+
+        Args:
+            entity_uri: URI of the entity (with or without angle brackets).
+
+        Returns:
+            Natural language description of the entity.
+        """
+        # Clean URI
+        entity_uri = entity_uri.strip()
+        if entity_uri.startswith('<') and entity_uri.endswith('>'):
+            entity_uri = entity_uri[1:-1]
+
+        # Query triples for this entity
+        query = f"""
+        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+        PREFIX dblp: <https://dblp.org/rdf/schema#>
+        SELECT ?predicate ?object
+        WHERE {{
+            <{entity_uri}> ?predicate ?object .
+        }}
+        """
+
+        try:
+            triples = execute_sparql_query(query, config=self.config)
+            return self.descriptor_function(entity_uri, triples)
+        except Exception as e:
+            print(f"Warning: Error describing entity {entity_uri}: {e}")
+            return _uri_to_label(entity_uri)
+
+    def describe_batch(self, entity_uris: List[str]) -> Dict[str, str]:
+        """Create descriptions for multiple entities in batch (optimized).
+
+        Uses a single SPARQL query with VALUES clause to fetch all triples,
+        then applies descriptor logic to each entity. Batch size limited to
+        80 entities to stay within 8KB SPARQL query size limit.
+
+        Args:
+            entity_uris: List of entity URIs (with or without angle brackets).
+
+        Returns:
+            Dictionary mapping original URIs to their descriptions.
+        """
+        if not entity_uris:
+            return {}
+
+        # Clean URIs
+        cleaned_uris = []
+        for uri in entity_uris:
+            uri = uri.strip()
+            if not uri:
+                continue
+            if uri.startswith('<') and uri.endswith('>'):
+                uri = uri[1:-1]
+            cleaned_uris.append(uri)
+
+        if not cleaned_uris:
+            return {}
+
+        # Build VALUES clause (limited to 80 URIs to stay within 8KB query limit)
+        if len(cleaned_uris) > 80:
+            print(f"Warning: Batch size {len(cleaned_uris)} exceeds limit of 80. "
+                  f"Consider splitting into multiple batches.")
+            cleaned_uris = cleaned_uris[:80]
+
+        values_clause = " ".join([f"<{uri}>" for uri in cleaned_uris])
+
+        query = f"""
+        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+        PREFIX dblp: <https://dblp.org/rdf/schema#>
+
+        SELECT ?entity ?predicate ?object
+        WHERE {{
+            VALUES ?entity {{ {values_clause} }}
+            ?entity ?predicate ?object .
+        }}
+        """
+
+        try:
+            triples = execute_sparql_query(query, config=self.config)
+        except Exception as e:
+            print(f"Warning: Error fetching triples for batch: {e}")
+            # Fallback to URI labels
+            return {uri: _uri_to_label(uri) for uri in entity_uris}
+
+        # Group triples by entity
+        entity_triples = {}
+        for triple in triples:
+            entity = triple['entity']
+            if entity not in entity_triples:
+                entity_triples[entity] = []
+            entity_triples[entity].append(triple)
+
+        # Create descriptions using descriptor function
+        descriptions = {}
+        for original_uri, cleaned_uri in zip(entity_uris[:len(cleaned_uris)], cleaned_uris):
+            if cleaned_uri not in entity_triples:
+                descriptions[original_uri] = _uri_to_label(original_uri)
+            else:
+                try:
+                    descriptions[original_uri] = self.descriptor_function(
+                        cleaned_uri,
+                        entity_triples[cleaned_uri]
+                    )
+                except Exception as e:
+                    print(f"Warning: Error applying descriptor function to {cleaned_uri}: {e}")
+                    descriptions[original_uri] = _uri_to_label(original_uri)
+
+        return descriptions
+
+
+if __name__ == "__main__":
+    # print(create_entity_description("https://dblp.org/rdf/schema#Publication"))
+    print(create_entity_descriptions_batch(["<https://dblp.org/rec/journals/nature/RheinbayNAWSTHH20>", "https://dblp.org/rdf/schema#Person"]))
+    # print(create_relation_description("http://www.w3.org/2000/01/rdf-schema#comment"))
+
+    # Example usage:
+    # entities = ["http://example.org/entity1", "http://example.org/entity2", "http://example.org/entity3"]
+    # descriptions = create_entity_descriptions_batch(entities)
+    #
+    # for entity, desc in descriptions.items():
+    #     print(f"{entity}: {desc}")

kgnode/_node_ranker.py

@@ -0,0 +1,138 @@
+import csv
+import os
+from functools import lru_cache
+from typing import List, Dict, Optional
+import time
+import threading
+from kgnode.core.kg_config import KGConfig
+
+from kgnode.core.sparql_query import execute_sparql_query
+
+@lru_cache(maxsize=10)
+def get_top_entities_by_degree(
+    limit: int = 1_000_000,
+    output_file: Optional[str] = None,
+    config: Optional[KGConfig] = None
+) -> List[Dict[str, str]]:
+    """
+    Get top N entities from knowledge graph sorted by degree (number of connections).
+    Saves results to CSV file.
+
+    Args:
+        limit (int): Number of top entities to retrieve. Default is 1,000,000.
+        output_file (str): Path to output CSV file. If None, defaults to ~/.kgnode/data/top_entities.csv.
+            Can be overridden via KGNODE_DATA_DIR environment variable.
+        config: Optional KGConfig instance for configuration.
+            If None, uses default KGConfig with environment variables or built-in defaults.
+
+    Returns:
+        List[Dict]: List of entities with their URIs and degrees.
+                   Each dict has keys: 'entity', 'degree'
+    """
+    # Initialize config if not provided
+    if config is None:
+        config = KGConfig.default()
+
+    # Use default path if not provided
+    if output_file is None:
+        output_file = config.csv_path
+
+    # Ensure parent directory exists
+    os.makedirs(os.path.dirname(output_file), exist_ok=True)
+
+    print(f"Querying top {limit:,} entities by degree, For 1 million nodes KG it takes 7 seconds to run.")
+
+    sparql_query = f"""
+    SELECT ?entity (COUNT(?o) as ?degree)
+        WHERE {{
+          ?entity ?p ?o .
+        }}
+        GROUP BY ?entity
+        ORDER BY DESC(?degree)
+        LIMIT {limit}
+    """
+
+    # in+out
+    # SELECT ?entity (COUNT(?connection) as ?degree)
+    #     WHERE {{
+    #       {{ ?entity ?p ?o }}
+    #       UNION
+    #       {{ ?s ?p ?entity }}
+    #     }}
+    #     GROUP BY ?entity
+    #     ORDER BY DESC(?degree)
+    #     LIMIT {limit}
+
+    # indegree
+    # SELECT ?entity (COUNT(?s) as ?degree)
+    # WHERE {{
+    #   ?s ?p ?entity .
+    # }}
+    # GROUP BY ?entity
+    # ORDER BY DESC(?degree)
+    # LIMIT {limit}
+
+    # outdegree
+    # SELECT ?entity (COUNT(?o) as ?degree)
+    #     WHERE {{
+    #       ?entity ?p ?o .
+    #     }}
+    #     GROUP BY ?entity
+    #     ORDER BY DESC(?degree)
+    #     LIMIT {limit}
+
+    # Spinner setup
+    spinner_chars = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
+    spinner_running = True
+
+    def spin():
+        i = 0
+        start = time.time()
+        while spinner_running:
+            elapsed = int(time.time() - start)
+            mins, secs = divmod(elapsed, 60)
+            print(f'\r{spinner_chars[i % len(spinner_chars)]} Querying... {mins:02d}:{secs:02d}', end='', flush=True)
+            i += 1
+            time.sleep(0.1)
+
+    # Start spinner
+    spinner_thread = threading.Thread(target=spin)
+    spinner_thread.start()
+
+    start_time = time.time()
+    results = execute_sparql_query(sparql_query, config=config)
+    query_time = time.time() - start_time
+
+    # Stop spinner
+    spinner_running = False
+    spinner_thread.join()
+
+    print(f"\r✓ Query completed in {query_time:.1f} seconds ({query_time / 60:.1f} minutes)")
+    print(f"✓ Retrieved {len(results):,} entities")
+    print(f"Saving to {output_file}...")
+
+    # Save to CSV
+    with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
+        fieldnames = ['entity', 'degree']
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        writer.writeheader()
+
+        for idx, row in enumerate(results):
+            writer.writerow({
+                'entity': row['entity'],
+                'degree': row['degree']
+            })
+
+            if (idx + 1) % 100_000 == 0:
+                print(f"  Written {idx + 1:,} rows...")
+
+    total_time = time.time() - start_time
+    print(f"✓ Done! Saved {len(results):,} entities to {output_file}")
+    print(f"Total time: {total_time:.1f} seconds")
+
+    return results
+
+
+if __name__ == "__main__":
+    entities = get_top_entities_by_degree(limit=10000)
+    print(entities)