resolvekit 0.0.1__py3-none-any.whl

resolvekit/utils/logging.py

@@ -0,0 +1,219 @@
+"""Logging utilities for resolvekit."""
+
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+from resolvekit.constants import LOG_DATE_FORMAT, LOG_FORMAT
+
+
+class StructuredFormatter(logging.Formatter):
+    """JSON structured log formatter."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format log record as JSON."""
+        log_data = {
+            "timestamp": self.formatTime(record, self.datefmt),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+
+        # Add extra fields
+        if hasattr(record, "extra"):
+            log_data.update(record.extra)
+
+        # Add exception info if present
+        if record.exc_info:
+            log_data["exception"] = self.formatException(record.exc_info)
+
+        return json.dumps(log_data)
+
+
+class PrivacyFilter(logging.Filter):
+    """Filter to prevent logging sensitive query content by default."""
+
+    def __init__(self, redact_queries: bool = True):
+        """Initialize privacy filter."""
+        super().__init__()
+        self.redact_queries = redact_queries
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        """Redact query content if privacy is enabled."""
+        if self.redact_queries and hasattr(record, "query"):
+            record.query = "[REDACTED]"
+        return True
+
+
+def setup_logging(
+    level: str | int = logging.INFO,
+    format_type: str = "human",
+    log_file: Path | str | None = None,
+    redact_queries: bool = True,
+) -> None:
+    """
+    Set up logging configuration.
+
+    Args:
+        level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+        format_type: "human" for human-readable, "json" for structured JSON
+        log_file: Optional file path for logging to file
+        redact_queries: Whether to redact query content (default: True for privacy)
+    """
+    # Convert string level to int
+    if isinstance(level, str):
+        level = getattr(logging, level.upper())
+
+    # Create handlers
+    handlers: list[logging.Handler] = []
+
+    # Console handler
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(level)
+
+    if format_type == "json":
+        console_handler.setFormatter(StructuredFormatter(datefmt=LOG_DATE_FORMAT))
+    else:
+        console_handler.setFormatter(logging.Formatter(LOG_FORMAT, LOG_DATE_FORMAT))
+
+    handlers.append(console_handler)
+
+    # File handler if specified
+    if log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setLevel(level)
+        if format_type == "json":
+            file_handler.setFormatter(StructuredFormatter(datefmt=LOG_DATE_FORMAT))
+        else:
+            file_handler.setFormatter(logging.Formatter(LOG_FORMAT, LOG_DATE_FORMAT))
+        handlers.append(file_handler)
+
+    # Configure root logger
+    root_logger = logging.getLogger()
+    root_logger.setLevel(level)
+
+    # Remove existing handlers
+    for handler in root_logger.handlers[:]:
+        root_logger.removeHandler(handler)
+
+    # Add new handlers
+    for handler in handlers:
+        root_logger.addHandler(handler)
+
+    # Add privacy filter
+    privacy_filter = PrivacyFilter(redact_queries=redact_queries)
+    for handler in handlers:
+        handler.addFilter(privacy_filter)
+
+
+def get_logger(name: str) -> logging.Logger:
+    """
+    Get a logger instance.
+
+    Args:
+        name: Logger name (typically __name__)
+
+    Returns:
+        Logger instance
+    """
+    return logging.getLogger(name)
+
+
+def log_performance(
+    logger: logging.Logger,
+    operation: str,
+    duration_ms: float,
+    extra: dict[str, Any] | None = None,
+) -> None:
+    """
+    Log performance metrics.
+
+    Args:
+        logger: Logger instance
+        operation: Operation name
+        duration_ms: Duration in milliseconds
+        extra: Additional context to log
+    """
+    log_data = {
+        "operation": operation,
+        "duration_ms": round(duration_ms, 2),
+    }
+    if extra:
+        log_data.update(extra)
+
+    logger.info(f"Performance: {operation} took {duration_ms:.2f}ms", extra=log_data)
+
+
+def log_resolution(
+    logger: logging.Logger,
+    query: str,
+    result_dcid: str | None,
+    confidence: float | None,
+    num_candidates: int,
+    duration_ms: float,
+    redact_query: bool = True,
+) -> None:
+    """
+    Log resolution operation.
+
+    Args:
+        logger: Logger instance
+        query: Query string (will be redacted if redact_query=True)
+        result_dcid: Resolved entity DCID (None if no match)
+        confidence: Confidence score
+        num_candidates: Number of candidates found
+        duration_ms: Resolution duration in milliseconds
+        redact_query: Whether to redact query content
+    """
+    log_data = {
+        "query": "[REDACTED]" if redact_query else query,
+        "result_dcid": result_dcid,
+        "confidence": round(confidence, 3) if confidence else None,
+        "num_candidates": num_candidates,
+        "duration_ms": round(duration_ms, 2),
+        "success": result_dcid is not None,
+    }
+
+    if result_dcid:
+        # Format confidence if present, otherwise indicate it's missing
+        conf_str = f"{confidence:.3f}" if confidence is not None else "unknown"
+        logger.info(
+            f"Resolved query to {result_dcid} (confidence={conf_str})",
+            extra=log_data,
+        )
+    else:
+        logger.warning("No match found for query", extra=log_data)
+
+
+def log_stage(
+    logger: logging.Logger,
+    stage: str,
+    candidates_before: int,
+    candidates_after: int,
+    duration_ms: float | None = None,
+) -> None:
+    """
+    Log resolution stage.
+
+    Args:
+        logger: Logger instance
+        stage: Stage name
+        candidates_before: Number of candidates before stage
+        candidates_after: Number of candidates after stage
+        duration_ms: Stage duration in milliseconds
+    """
+    log_data = {
+        "stage": stage,
+        "candidates_before": candidates_before,
+        "candidates_after": candidates_after,
+        "filtered": candidates_before - candidates_after,
+    }
+    if duration_ms is not None:
+        log_data["duration_ms"] = round(duration_ms, 2)
+
+    logger.debug(
+        f"Stage '{stage}': {candidates_before} → {candidates_after} candidates",
+        extra=log_data,
+    )

resolvekit/utils/text.py

@@ -0,0 +1,475 @@
+"""Text processing utilities for resolvekit.
+
+This module uses rapidfuzz for efficient fuzzy string matching.
+We keep our normalization functions and wrap rapidfuzz for consistency.
+"""
+
+import unicodedata
+from collections.abc import Iterable
+
+from rapidfuzz import distance, fuzz, process
+
+
+def normalize_unicode(text: str, form: str = "NFKD") -> str:
+    """
+    Normalize Unicode text.
+
+    Args:
+        text: Input text
+        form: Normalization form (NFC, NFD, NFKC, NFKD)
+
+    Returns:
+        Normalized text
+
+    Examples:
+        >>> normalize_unicode("café")
+        'café'  # NFKD form
+    """
+    return unicodedata.normalize(form, text)
+
+
+def remove_diacritics(text: str) -> str:
+    """
+    Remove diacritical marks from text.
+
+    Args:
+        text: Input text
+
+    Returns:
+        Text without diacritics
+
+    Examples:
+        >>> remove_diacritics("Côte d'Ivoire")
+        "Cote d'Ivoire"
+        >>> remove_diacritics("Türkiye")
+        "Turkiye"
+    """
+    # Normalize to NFD (decomposed form)
+    nfd = unicodedata.normalize("NFD", text)
+    # Filter out combining characters (diacritics)
+    return "".join(char for char in nfd if unicodedata.category(char) != "Mn")
+
+
+def case_fold(text: str) -> str:
+    """
+    Case fold text for case-insensitive matching.
+
+    Args:
+        text: Input text
+
+    Returns:
+        Case-folded text (lowercase with special handling)
+
+    Examples:
+        >>> case_fold("FRANCE")
+        'france'
+        >>> case_fold("Türkiye")
+        'türkiye'
+    """
+    return text.casefold()
+
+
+def normalize_whitespace(text: str) -> str:
+    """
+    Normalize whitespace in text.
+
+    Args:
+        text: Input text
+
+    Returns:
+        Text with normalized whitespace
+
+    Examples:
+        >>> normalize_whitespace("  hello   world  ")
+        'hello world'
+    """
+    import re
+
+    # Replace multiple spaces with single space
+    text = re.sub(r"\s+", " ", text)
+    # Strip leading/trailing whitespace
+    return text.strip()
+
+
+def edit_distance(s1: str, s2: str) -> int:
+    """
+    Compute Levenshtein edit distance between two strings.
+
+    Powered by rapidfuzz for performance.
+
+    Args:
+        s1: First string
+        s2: Second string
+
+    Returns:
+        Edit distance (number of insertions, deletions, substitutions)
+
+    Examples:
+        >>> edit_distance("kitten", "sitting")
+        3
+        >>> edit_distance("france", "frence")
+        1
+    """
+    return distance.Levenshtein.distance(s1, s2)
+
+
+def normalized_edit_distance(s1: str, s2: str) -> float:
+    """
+    Compute normalized edit distance (0.0 = identical, 1.0 = completely different).
+
+    Powered by rapidfuzz for performance.
+
+    Args:
+        s1: First string
+        s2: Second string
+
+    Returns:
+        Normalized edit distance [0.0, 1.0]
+
+    Examples:
+        >>> normalized_edit_distance("france", "france")
+        0.0
+        >>> normalized_edit_distance("abc", "xyz")
+        1.0
+    """
+    return distance.Levenshtein.normalized_distance(s1, s2)
+
+
+def string_similarity(s1: str, s2: str) -> float:
+    """
+    Compute overall string similarity score.
+
+    Uses rapidfuzz's ratio algorithm (similar to difflib.SequenceMatcher).
+
+    Args:
+        s1: First string
+        s2: Second string
+
+    Returns:
+        Similarity score [0.0, 1.0] (1.0 = identical)
+
+    Examples:
+        >>> string_similarity("france", "france")
+        1.0
+        >>> string_similarity("france", "frence")
+        0.83  # Approximately
+    """
+    return fuzz.ratio(s1, s2) / 100.0
+
+
+def trigram_similarity(s1: str, s2: str) -> float:
+    """
+    Compute trigram similarity between two strings.
+
+    Uses rapidfuzz's token_set_ratio for better performance.
+
+    Args:
+        s1: First string
+        s2: Second string
+
+    Returns:
+        Similarity score [0.0, 1.0] (1.0 = identical)
+
+    Examples:
+        >>> trigram_similarity("france", "france")
+        1.0
+        >>> trigram_similarity("germany", "germeny")
+        # High similarity but not identical
+    """
+    # Use token_set_ratio which is similar to trigram matching
+    return fuzz.token_set_ratio(s1, s2) / 100.0
+
+
+def partial_similarity(s1: str, s2: str) -> float:
+    """
+    Compute partial string similarity (substring matching).
+
+    Useful when one string contains the other.
+
+    Args:
+        s1: First string
+        s2: Second string
+
+    Returns:
+        Similarity score [0.0, 1.0]
+
+    Examples:
+        >>> partial_similarity("france", "southern france")
+        1.0
+    """
+    return fuzz.partial_ratio(s1, s2) / 100.0
+
+
+def jaccard_similarity(set1: Iterable, set2: Iterable) -> float:
+    """
+    Compute Jaccard similarity between two sets.
+
+    Args:
+        set1: First set (or iterable)
+        set2: Second set (or iterable)
+
+    Returns:
+        Jaccard similarity [0.0, 1.0]
+
+    Examples:
+        >>> jaccard_similarity({1, 2, 3}, {2, 3, 4})
+        0.5
+    """
+    s1 = set(set1)
+    s2 = set(set2)
+
+    if not s1 and not s2:
+        return 1.0
+
+    intersection = len(s1 & s2)
+    union = len(s1 | s2)
+
+    return intersection / union if union > 0 else 0.0
+
+
+def token_jaccard_similarity(s1: str, s2: str) -> float:
+    """
+    Compute Jaccard similarity based on tokens (whitespace-separated words).
+
+    Args:
+        s1: First string
+        s2: Second string
+
+    Returns:
+        Token-based Jaccard similarity [0.0, 1.0]
+
+    Examples:
+        >>> token_jaccard_similarity("hello world", "world hello")
+        1.0
+        >>> token_jaccard_similarity("democratic republic of congo", "republic of congo")
+        0.75
+    """
+    tokens1 = set(s1.lower().split())
+    tokens2 = set(s2.lower().split())
+    return jaccard_similarity(tokens1, tokens2)
+
+
+def find_best_match(
+    query: str, choices: list[str], limit: int = 1
+) -> list[tuple[str, float]]:
+    """
+    Find best matching string(s) from a list of choices.
+
+    Powered by rapidfuzz for efficient searching.
+
+    Args:
+        query: Query string
+        choices: List of candidate strings
+        limit: Number of results to return
+
+    Returns:
+        List of (match, score) tuples, sorted by score descending
+
+    Examples:
+        >>> find_best_match("frence", ["france", "germany", "italy"])
+        [('france', 0.83)]
+    """
+    if not choices:
+        return []
+
+    results = process.extract(query, choices, scorer=fuzz.ratio, limit=limit)
+    # Convert scores from 0-100 to 0-1
+    return [(match, score / 100.0) for match, score, _ in results]
+
+
+def starts_with_prefix(text: str, prefix: str, case_sensitive: bool = False) -> bool:
+    """
+    Check if text starts with prefix.
+
+    Args:
+        text: Text to check
+        prefix: Prefix to look for
+        case_sensitive: Whether to use case-sensitive matching
+
+    Returns:
+        True if text starts with prefix
+    """
+    if not case_sensitive:
+        text = text.lower()
+        prefix = prefix.lower()
+
+    return text.startswith(prefix)
+
+
+def contains_substring(text: str, substring: str, case_sensitive: bool = False) -> bool:
+    """
+    Check if text contains substring.
+
+    Args:
+        text: Text to search in
+        substring: Substring to look for
+        case_sensitive: Whether to use case-sensitive matching
+
+    Returns:
+        True if substring is found
+    """
+    if not case_sensitive:
+        text = text.lower()
+        substring = substring.lower()
+
+    return substring in text
+
+
+def truncate(text: str, max_length: int, suffix: str = "...") -> str:
+    """
+    Truncate text to maximum length.
+
+    Args:
+        text: Text to truncate
+        max_length: Maximum length (including suffix)
+        suffix: Suffix to append when truncating
+
+    Returns:
+        Truncated text
+
+    Examples:
+        >>> truncate("hello world", 8)
+        'hello...'
+    """
+    if len(text) <= max_length:
+        return text
+
+    return text[: max_length - len(suffix)] + suffix
+
+
+def get_common_prefix(s1: str, s2: str) -> str:
+    """
+    Get common prefix of two strings.
+
+    Args:
+        s1: First string
+        s2: Second string
+
+    Returns:
+        Common prefix
+
+    Examples:
+        >>> get_common_prefix("france", "french")
+        'fr'
+    """
+    prefix = []
+    for c1, c2 in zip(s1, s2, strict=False):
+        if c1 == c2:
+            prefix.append(c1)
+        else:
+            break
+    return "".join(prefix)
+
+
+def get_common_suffix(s1: str, s2: str) -> str:
+    """
+    Get common suffix of two strings.
+
+    Args:
+        s1: First string
+        s2: Second string
+
+    Returns:
+        Common suffix
+
+    Examples:
+        >>> get_common_suffix("testing", "running")
+        'ing'
+    """
+    return get_common_prefix(s1[::-1], s2[::-1])[::-1]
+
+
+def is_acronym_of(short: str, long: str) -> bool:
+    """
+    Check if short string could be an acronym of long string.
+
+    Args:
+        short: Potential acronym
+        long: Full form
+
+    Returns:
+        True if short could be acronym of long
+
+    Examples:
+        >>> is_acronym_of("USA", "United States of America")
+        True
+        >>> is_acronym_of("UN", "United Nations")
+        True
+        >>> is_acronym_of("XYZ", "United Nations")
+        False
+    """
+    short = short.upper()
+    words = long.upper().split()
+
+    if len(short) > len(words):
+        return False
+
+    # Check if each letter in short matches first letter of words
+    if len(short) == len(words):
+        return all(s == w[0] for s, w in zip(short, words, strict=False))
+
+    # More complex matching (skip some words)
+    # Try to match acronym letters to word first letters
+    word_idx = 0
+    for char in short:
+        # Find next word starting with this character
+        while word_idx < len(words) and words[word_idx][0] != char:
+            word_idx += 1
+        if word_idx >= len(words):
+            return False
+        word_idx += 1
+
+    return True
+
+
+def clean_text(text: str, remove_diacritics: bool = False) -> str:
+    """
+    Clean text for matching (normalize unicode, whitespace, optionally remove diacritics).
+
+    Args:
+        text: Input text
+        remove_diacritics: Whether to remove diacritical marks
+
+    Returns:
+        Cleaned text
+
+    Examples:
+        >>> clean_text("  Côte  d'Ivoire  ", remove_diacritics=True)
+        "cote d'ivoire"
+    """
+    # Normalize unicode
+    text = normalize_unicode(text)
+
+    # Optionally remove diacritics
+    if remove_diacritics:
+        # Call the remove_diacritics function
+        text = globals()["remove_diacritics"](text)
+
+    # Case fold
+    text = case_fold(text)
+
+    # Normalize whitespace
+    text = normalize_whitespace(text)
+
+    return text
+
+
+# Keep get_trigrams for backward compatibility
+def get_trigrams(text: str) -> set[str]:
+    """
+    Extract trigrams from text.
+
+    Args:
+        text: Input text
+
+    Returns:
+        Set of trigrams
+
+    Examples:
+        >>> get_trigrams("hello")
+        {'hel', 'ell', 'llo'}
+    """
+    if len(text) < 3:
+        return {text}
+
+    return {text[i : i + 3] for i in range(len(text) - 2)}