aiecs 1.0.1__py3-none-any.whl → 1.7.6__py3-none-any.whl

aiecs/application/knowledge_graph/search/text_similarity.py

@@ -0,0 +1,392 @@
+"""
+Text Similarity Utilities
+
+Provides various text similarity and matching functions for knowledge graph operations.
+Includes BM25, Jaccard, cosine similarity, Levenshtein distance, and fuzzy matching.
+"""
+
+import re
+import math
+from typing import List, Optional, Tuple, Callable, Any
+from collections import Counter
+from difflib import SequenceMatcher
+
+
+class BM25Scorer:
+    """
+    BM25 (Best Matching 25) scorer for text similarity
+
+    BM25 is a ranking function used to estimate the relevance of documents
+    to a given search query. It's an improvement over TF-IDF.
+
+    Example::
+
+        scorer = BM25Scorer(corpus=[
+            "The quick brown fox jumps over the lazy dog",
+            "A quick brown dog jumps over a lazy fox",
+            "The lazy dog sleeps all day"
+        ])
+
+        scores = scorer.score("quick brown fox")
+        # Returns scores for each document in corpus
+    """
+
+    def __init__(
+        self,
+        corpus: List[str],
+        k1: float = 1.5,
+        b: float = 0.75,
+        tokenizer: Optional[Callable[[str], List[str]]] = None,
+    ):
+        """
+        Initialize BM25 scorer
+
+        Args:
+            corpus: List of documents to score against
+            k1: Term frequency saturation parameter (default: 1.5)
+            b: Length normalization parameter (default: 0.75)
+            tokenizer: Optional tokenizer function (default: simple word split)
+        """
+        self.k1 = k1
+        self.b = b
+        self.tokenizer = tokenizer or self._default_tokenizer
+
+        # Tokenize corpus
+        self.documents = [self.tokenizer(doc) for doc in corpus]
+        self.doc_count = len(self.documents)
+
+        # Calculate document lengths
+        self.doc_lengths = [len(doc) for doc in self.documents]
+        self.avg_doc_length = sum(self.doc_lengths) / self.doc_count if self.doc_count > 0 else 0
+
+        # Build term frequency dictionary
+        self.term_freqs = []
+        self.doc_freqs: Counter[str] = Counter()
+
+        for doc in self.documents:
+            tf = Counter(doc)
+            self.term_freqs.append(tf)
+            for term in set(doc):
+                self.doc_freqs[term] += 1
+
+        # Calculate IDF (Inverse Document Frequency)
+        self.idf = {}
+        for term, df in self.doc_freqs.items():
+            self.idf[term] = math.log((self.doc_count - df + 0.5) / (df + 0.5) + 1.0)
+
+    def _default_tokenizer(self, text: str) -> List[str]:
+        """Default tokenizer: lowercase and split on whitespace"""
+        return re.findall(r"\w+", text.lower())
+
+    def score(self, query: str) -> List[float]:
+        """
+        Score documents against query
+
+        Args:
+            query: Query string
+
+        Returns:
+            List of BM25 scores for each document
+        """
+        query_terms = self.tokenizer(query)
+        scores = []
+
+        for i, doc in enumerate(self.documents):
+            score = 0.0
+            doc_length = self.doc_lengths[i]
+            term_freq = self.term_freqs[i]
+
+            for term in query_terms:
+                if term in term_freq:
+                    tf = term_freq[term]
+                    idf = self.idf.get(term, 0.0)
+
+                    # BM25 formula
+                    numerator = idf * tf * (self.k1 + 1)
+                    denominator = tf + self.k1 * (1 - self.b + self.b * (doc_length / self.avg_doc_length))
+                    score += numerator / denominator
+
+            scores.append(score)
+
+        return scores
+
+    def get_top_n(self, query: str, n: int = 10) -> List[Tuple[int, float]]:
+        """
+        Get top N documents by BM25 score
+
+        Args:
+            query: Query string
+            n: Number of top results to return
+
+        Returns:
+            List of (document_index, score) tuples, sorted by score descending
+        """
+        scores = self.score(query)
+        indexed_scores = [(i, score) for i, score in enumerate(scores)]
+        indexed_scores.sort(key=lambda x: x[1], reverse=True)
+        return indexed_scores[:n]
+
+
+def jaccard_similarity(set1: set, set2: set) -> float:
+    """
+    Calculate Jaccard similarity between two sets
+
+    Jaccard similarity = (size of intersection) / (size of union)
+
+    Args:
+        set1: First set
+        set2: Second set
+
+    Returns:
+        Jaccard similarity score (0.0 to 1.0)
+    """
+    if not set1 and not set2:
+        return 1.0
+
+    intersection = len(set1 & set2)
+    union = len(set1 | set2)
+
+    if union == 0:
+        return 0.0
+
+    return intersection / union
+
+
+def jaccard_similarity_text(text1: str, text2: str, tokenizer: Optional[Callable[[str], Any]] = None) -> float:
+    """
+    Calculate Jaccard similarity between two text strings
+
+    Args:
+        text1: First text string
+        text2: Second text string
+        tokenizer: Optional tokenizer function (default: word split)
+
+    Returns:
+        Jaccard similarity score (0.0 to 1.0)
+    """
+    if tokenizer is None:
+
+        def tokenizer(t):
+            return set(re.findall(r"\w+", t.lower()))
+
+    else:
+        # Wrap tokenizer to ensure it returns a set
+        original_tokenizer = tokenizer
+
+        def tokenizer(t):
+            return set(original_tokenizer(t))
+
+    set1 = tokenizer(text1)
+    set2 = tokenizer(text2)
+
+    return jaccard_similarity(set1, set2)
+
+
+def cosine_similarity_text(text1: str, text2: str, tokenizer: Optional[Callable[[str], List[str]]] = None) -> float:
+    """
+    Calculate cosine similarity between two text strings
+
+    Cosine similarity measures the cosine of the angle between two vectors
+    in a multi-dimensional space. For text, vectors are TF-IDF representations.
+
+    Args:
+        text1: First text string
+        text2: Second text string
+        tokenizer: Optional tokenizer function (default: word split)
+
+    Returns:
+        Cosine similarity score (0.0 to 1.0)
+    """
+    if tokenizer is None:
+
+        def tokenizer(t):
+            return re.findall(r"\w+", t.lower())
+
+    tokens1 = tokenizer(text1)
+    tokens2 = tokenizer(text2)
+
+    # Build vocabulary
+    vocab = set(tokens1) | set(tokens2)
+
+    if not vocab:
+        return 1.0 if not text1 and not text2 else 0.0
+
+    # Create term frequency vectors
+    tf1 = Counter(tokens1)
+    tf2 = Counter(tokens2)
+
+    # Calculate dot product and magnitudes
+    dot_product = sum(tf1.get(term, 0) * tf2.get(term, 0) for term in vocab)
+    magnitude1 = math.sqrt(sum(tf1.get(term, 0) ** 2 for term in vocab))
+    magnitude2 = math.sqrt(sum(tf2.get(term, 0) ** 2 for term in vocab))
+
+    if magnitude1 == 0 or magnitude2 == 0:
+        return 0.0
+
+    similarity = dot_product / (magnitude1 * magnitude2)
+    # Handle floating point precision issues
+    return min(1.0, max(0.0, similarity))
+
+
+def levenshtein_distance(s1: str, s2: str) -> int:
+    """
+    Calculate Levenshtein distance (edit distance) between two strings
+
+    Levenshtein distance is the minimum number of single-character edits
+    (insertions, deletions, or substitutions) required to change one string
+    into another.
+
+    Args:
+        s1: First string
+        s2: Second string
+
+    Returns:
+        Levenshtein distance (0 = identical, higher = more different)
+    """
+    if len(s1) < len(s2):
+        return levenshtein_distance(s2, s1)
+
+    if len(s2) == 0:
+        return len(s1)
+
+    # Use dynamic programming
+    previous_row = list(range(len(s2) + 1))
+
+    for i, c1 in enumerate(s1):
+        current_row = [i + 1]
+        for j, c2 in enumerate(s2):
+            insertions = previous_row[j + 1] + 1
+            deletions = current_row[j] + 1
+            substitutions = previous_row[j] + (c1 != c2)
+            current_row.append(min(insertions, deletions, substitutions))
+        previous_row = current_row
+
+    return previous_row[-1]
+
+
+def normalized_levenshtein_similarity(s1: str, s2: str) -> float:
+    """
+    Calculate normalized Levenshtein similarity (0.0 to 1.0)
+
+    Args:
+        s1: First string
+        s2: Second string
+
+    Returns:
+        Normalized similarity score (1.0 = identical, 0.0 = completely different)
+    """
+    max_len = max(len(s1), len(s2))
+    if max_len == 0:
+        return 1.0
+
+    distance = levenshtein_distance(s1, s2)
+    return 1.0 - (distance / max_len)
+
+
+def fuzzy_match(
+    query: str,
+    candidates: List[str],
+    threshold: float = 0.6,
+    method: str = "jaccard",
+) -> List[Tuple[str, float]]:
+    """
+    Find fuzzy matches for a query string in a list of candidates
+
+    Args:
+        query: Query string to match
+        candidates: List of candidate strings
+        threshold: Minimum similarity threshold (0.0 to 1.0)
+        method: Similarity method ("jaccard", "cosine", "levenshtein", "ratio")
+
+    Returns:
+        List of (candidate, similarity_score) tuples above threshold,
+        sorted by score descending
+    """
+    results = []
+
+    for candidate in candidates:
+        if method == "jaccard":
+            score = jaccard_similarity_text(query, candidate)
+        elif method == "cosine":
+            score = cosine_similarity_text(query, candidate)
+        elif method == "levenshtein":
+            score = normalized_levenshtein_similarity(query, candidate)
+        elif method == "ratio":
+            # Use SequenceMatcher ratio (built-in fuzzy matching)
+            score = SequenceMatcher(None, query.lower(), candidate.lower()).ratio()
+        else:
+            raise ValueError(f"Unknown method: {method}. Use 'jaccard', 'cosine', 'levenshtein', or 'ratio'")
+
+        if score >= threshold:
+            results.append((candidate, score))
+
+    # Sort by score descending
+    results.sort(key=lambda x: x[1], reverse=True)
+    return results
+
+
+class TextSimilarity:
+    """
+    Convenience class for text similarity operations
+
+    Provides a unified interface for various text similarity methods.
+
+    Example::
+
+        similarity = TextSimilarity()
+
+        # Jaccard similarity
+        score = similarity.jaccard("hello world", "world hello")
+
+        # Cosine similarity
+        score = similarity.cosine("machine learning", "deep learning")
+
+        # Levenshtein distance
+        distance = similarity.levenshtein("kitten", "sitting")
+
+        # Fuzzy matching
+        matches = similarity.fuzzy_match(
+            "python",
+            ["python3", "pyton", "java", "pythn"],
+            threshold=0.7
+        )
+    """
+
+    def __init__(self, tokenizer: Optional[Callable[[str], List[str]]] = None):
+        """
+        Initialize TextSimilarity
+
+        Args:
+            tokenizer: Optional tokenizer function for text processing
+        """
+        self.tokenizer = tokenizer
+
+    def jaccard(self, text1: str, text2: str) -> float:
+        """Calculate Jaccard similarity between two texts"""
+        return jaccard_similarity_text(text1, text2, self.tokenizer)
+
+    def cosine(self, text1: str, text2: str) -> float:
+        """Calculate cosine similarity between two texts"""
+        return cosine_similarity_text(text1, text2, self.tokenizer)
+
+    def levenshtein(self, text1: str, text2: str) -> int:
+        """Calculate Levenshtein distance between two texts"""
+        return levenshtein_distance(text1, text2)
+
+    def levenshtein_similarity(self, text1: str, text2: str) -> float:
+        """Calculate normalized Levenshtein similarity"""
+        return normalized_levenshtein_similarity(text1, text2)
+
+    def fuzzy_match(
+        self,
+        query: str,
+        candidates: List[str],
+        threshold: float = 0.6,
+        method: str = "jaccard",
+    ) -> List[Tuple[str, float]]:
+        """Find fuzzy matches for a query"""
+        return fuzzy_match(query, candidates, threshold, method)
+
+    def bm25(self, corpus: List[str], k1: float = 1.5, b: float = 0.75) -> BM25Scorer:
+        """Create a BM25 scorer for a corpus"""
+        return BM25Scorer(corpus, k1=k1, b=b, tokenizer=self.tokenizer)

aiecs/application/knowledge_graph/traversal/__init__.py

@@ -0,0 +1,15 @@
+"""
+Knowledge Graph Traversal Application Layer
+
+Advanced traversal algorithms and path ranking utilities.
+"""
+
+from aiecs.application.knowledge_graph.traversal.path_scorer import PathScorer
+from aiecs.application.knowledge_graph.traversal.enhanced_traversal import (
+    EnhancedTraversal,
+)
+
+__all__ = [
+    "PathScorer",
+    "EnhancedTraversal",
+]

aiecs/application/knowledge_graph/traversal/enhanced_traversal.py

@@ -0,0 +1,305 @@
+"""
+Enhanced Graph Traversal
+
+Provides advanced traversal capabilities with PathPattern support,
+cycle detection, and sophisticated path filtering.
+"""
+
+from typing import List, Optional
+from collections import deque
+from aiecs.domain.knowledge_graph.models.relation import Relation
+from aiecs.domain.knowledge_graph.models.path import Path
+from aiecs.domain.knowledge_graph.models.path_pattern import (
+    PathPattern,
+    TraversalDirection,
+)
+from aiecs.infrastructure.graph_storage.base import GraphStore
+
+
+class EnhancedTraversal:
+    """
+    Enhanced Graph Traversal Service
+
+    Provides advanced traversal capabilities beyond basic BFS:
+    - PathPattern-based traversal
+    - Cycle detection and handling
+    - Depth-limited traversal with constraints
+    - Path filtering by pattern
+
+    Example:
+        ```python
+        traversal = EnhancedTraversal(graph_store)
+
+        # Define pattern
+        pattern = PathPattern(
+            relation_types=["WORKS_FOR", "LOCATED_IN"],
+            max_depth=2,
+            allow_cycles=False
+        )
+
+        # Traverse with pattern
+        paths = await traversal.traverse_with_pattern(
+            start_entity_id="person_1",
+            pattern=pattern,
+            max_results=10
+        )
+        ```
+    """
+
+    def __init__(self, graph_store: GraphStore):
+        """
+        Initialize enhanced traversal service
+
+        Args:
+            graph_store: Graph storage backend to use
+        """
+        self.graph_store = graph_store
+
+    async def traverse_with_pattern(
+        self,
+        start_entity_id: str,
+        pattern: PathPattern,
+        max_results: int = 100,
+    ) -> List[Path]:
+        """
+        Traverse graph following a path pattern
+
+        Args:
+            start_entity_id: Starting entity ID
+            pattern: Path pattern to follow
+            max_results: Maximum number of paths to return
+
+        Returns:
+            List of paths matching the pattern
+        """
+        start_entity = await self.graph_store.get_entity(start_entity_id)
+        if start_entity is None:
+            return []
+
+        # Check if start entity is allowed
+        if not pattern.is_entity_allowed(start_entity.id, start_entity.entity_type):
+            return []
+
+        paths: List[Path] = []
+        # visited_in_path: Set[str] = set() if not pattern.allow_cycles else
+        # None  # Reserved for future use
+
+        # BFS with pattern matching
+        queue: deque = deque()
+        queue.append(
+            {
+                "entity": start_entity,
+                "path_entities": [start_entity],
+                "path_edges": [],
+                "depth": 0,
+                "visited": ({start_entity.id} if not pattern.allow_cycles else set()),
+            }
+        )
+
+        while queue and len(paths) < max_results:
+            current = queue.popleft()
+            current_entity = current["entity"]
+            current_depth = current["depth"]
+            path_entities = current["path_entities"]
+            path_edges = current["path_edges"]
+            visited_nodes = current["visited"]
+
+            # Add path if it meets length requirements
+            if pattern.is_valid_path_length(len(path_edges)):
+                path = Path(nodes=path_entities, edges=path_edges)
+                paths.append(path)
+
+            # Continue traversal if not at max depth
+            if not pattern.should_continue_traversal(current_depth):
+                continue
+
+            # Get neighbors based on pattern direction
+            # pattern.direction is already a string due to use_enum_values=True
+            direction_str = pattern.direction if isinstance(pattern.direction, str) else pattern.direction.value
+            neighbors = await self.graph_store.get_neighbors(
+                current_entity.id,
+                relation_type=None,  # We'll filter by pattern
+                direction=direction_str,
+            )
+
+            for neighbor in neighbors:
+                # Check if entity is allowed
+                if not pattern.is_entity_allowed(neighbor.id, neighbor.entity_type):
+                    continue
+
+                # Check for cycles
+                if not pattern.allow_cycles and neighbor.id in visited_nodes:
+                    continue
+
+                # Get the relation between current and neighbor
+                # We need to find the actual relation
+                relation = await self._find_relation(current_entity.id, neighbor.id, pattern.direction)
+
+                if relation is None:
+                    continue
+
+                # Check if relation is allowed at this depth
+                if not pattern.is_relation_allowed(relation.relation_type, current_depth):
+                    continue
+
+                # For incoming direction, we need to reverse the relation for path construction
+                # because paths expect edges[i].source_id == nodes[i].id
+                direction_str = pattern.direction if isinstance(pattern.direction, str) else pattern.direction.value
+                if direction_str == "incoming":
+                    # Reverse the relation: if we have e1->e2 and we're going from e2 to e1,
+                    # the path needs e2->e1 (source=current, target=neighbor)
+                    path_relation = Relation(
+                        id=f"{relation.id}_reversed",
+                        relation_type=relation.relation_type,
+                        source_id=current_entity.id,
+                        target_id=neighbor.id,
+                        weight=relation.weight,
+                    )
+                else:
+                    path_relation = relation
+
+                # Create new path state
+                new_path_entities = path_entities + [neighbor]
+                new_path_edges = path_edges + [path_relation]
+                new_visited = visited_nodes | {neighbor.id} if not pattern.allow_cycles else visited_nodes
+
+                queue.append(
+                    {
+                        "entity": neighbor,
+                        "path_entities": new_path_entities,
+                        "path_edges": new_path_edges,
+                        "depth": current_depth + 1,
+                        "visited": new_visited,
+                    }
+                )
+
+        return paths
+
+    async def _find_relation(self, source_id: str, target_id: str, direction: TraversalDirection) -> Optional[Relation]:
+        """
+        Find the relation between two entities
+
+        Args:
+            source_id: Source entity ID
+            target_id: Target entity ID
+            direction: Traversal direction (can be enum or string)
+
+        Returns:
+            Relation if found, None otherwise
+        """
+        # Try to find the actual relation in the graph store
+        # This works with both InMemoryGraphStore and SQLiteGraphStore
+
+        # Handle both enum and string directions
+        direction_str = direction if isinstance(direction, str) else direction.value
+        direction_enum = TraversalDirection(direction_str) if isinstance(direction, str) else direction
+
+        if direction_enum == TraversalDirection.OUTGOING or direction_enum == TraversalDirection.BOTH:
+            # Look for outgoing relations from source
+            neighbors = await self.graph_store.get_neighbors(source_id, relation_type=None, direction="outgoing")
+            for neighbor in neighbors:
+                if neighbor.id == target_id:
+                    # Found the neighbor, now get the relation
+                    # This is a workaround - ideally get_neighbors would return relations too
+                    # For now, check if the store exposes relations
+                    from aiecs.infrastructure.graph_storage.in_memory import (
+                        InMemoryGraphStore,
+                    )
+
+                    if isinstance(self.graph_store, InMemoryGraphStore):
+                        for rel in self.graph_store.relations.values():
+                            if rel.source_id == source_id and rel.target_id == target_id:
+                                return rel
+                    else:
+                        # For SQLite or other stores, try to get the relation
+                        # This is a placeholder - real implementation would
+                        # query the DB
+                        return Relation(
+                            id=f"rel_{source_id}_{target_id}",
+                            relation_type="CONNECTED_TO",
+                            source_id=source_id,
+                            target_id=target_id,
+                        )
+
+        if direction_enum == TraversalDirection.INCOMING or direction_enum == TraversalDirection.BOTH:
+            # Look for incoming relations to source (i.e., outgoing from
+            # target)
+            neighbors = await self.graph_store.get_neighbors(target_id, relation_type=None, direction="outgoing")
+            for neighbor in neighbors:
+                if neighbor.id == source_id:
+                    from aiecs.infrastructure.graph_storage.in_memory import (
+                        InMemoryGraphStore,
+                    )
+
+                    if isinstance(self.graph_store, InMemoryGraphStore):
+                        for rel in self.graph_store.relations.values():
+                            if rel.source_id == target_id and rel.target_id == source_id:
+                                return rel
+                    else:
+                        return Relation(
+                            id=f"rel_{target_id}_{source_id}",
+                            relation_type="CONNECTED_TO",
+                            source_id=target_id,
+                            target_id=source_id,
+                        )
+
+        return None
+
+    def detect_cycles(self, path: Path) -> bool:
+        """
+        Detect if a path contains cycles (repeated nodes)
+
+        Args:
+            path: Path to check
+
+        Returns:
+            True if path contains cycles
+        """
+        entity_ids = path.get_entity_ids()
+        return len(entity_ids) != len(set(entity_ids))
+
+    def filter_paths_without_cycles(self, paths: List[Path]) -> List[Path]:
+        """
+        Filter out paths that contain cycles
+
+        Args:
+            paths: List of paths to filter
+
+        Returns:
+            List of paths without cycles
+        """
+        return [path for path in paths if not self.detect_cycles(path)]
+
+    async def find_all_paths_between(
+        self,
+        source_id: str,
+        target_id: str,
+        pattern: Optional[PathPattern] = None,
+        max_paths: int = 10,
+    ) -> List[Path]:
+        """
+        Find all paths between two entities matching a pattern
+
+        Args:
+            source_id: Source entity ID
+            target_id: Target entity ID
+            pattern: Optional path pattern to follow
+            max_paths: Maximum number of paths to return
+
+        Returns:
+            List of paths from source to target
+        """
+        if pattern is None:
+            pattern = PathPattern(max_depth=5, allow_cycles=False)
+
+        # Traverse from source
+        all_paths = await self.traverse_with_pattern(
+            start_entity_id=source_id,
+            pattern=pattern,
+            max_results=max_paths * 10,  # Get more paths for filtering
+        )
+
+        # Filter paths that end at target
+        target_paths = [path for path in all_paths if path.end_entity.id == target_id]
+
+        return target_paths[:max_paths]