dataknobs-xization 1.2.3__py3-none-any.whl

dataknobs_xization/markdown/__init__.py

@@ -0,0 +1,72 @@
+"""Markdown chunking utilities for RAG applications.
+
+This module provides comprehensive utilities for parsing and chunking markdown
+documents while preserving semantic structure and heading hierarchy.
+"""
+
+from dataknobs_xization.markdown.md_chunker import (
+    Chunk,
+    ChunkFormat,
+    ChunkMetadata,
+    HeadingInclusion,
+    MarkdownChunker,
+    chunk_markdown_tree,
+)
+from dataknobs_xization.markdown.md_parser import (
+    MarkdownNode,
+    MarkdownParser,
+    parse_markdown,
+)
+from dataknobs_xization.markdown.md_streaming import (
+    AdaptiveStreamingProcessor,
+    StreamingMarkdownProcessor,
+    stream_markdown_file,
+    stream_markdown_string,
+)
+from dataknobs_xization.markdown.filters import (
+    ChunkQualityConfig,
+    ChunkQualityFilter,
+)
+from dataknobs_xization.markdown.enrichment import (
+    EnrichedChunkData,
+    build_enriched_text,
+    enrich_chunk,
+    extract_heading_metadata,
+    format_heading_display,
+    format_heading_for_display,
+    get_dynamic_heading_display,
+    get_relevant_headings_for_display,
+    is_multiword,
+)
+
+__all__ = [
+    # Parser
+    "MarkdownNode",
+    "MarkdownParser",
+    "parse_markdown",
+    # Chunker
+    "Chunk",
+    "ChunkFormat",
+    "ChunkMetadata",
+    "HeadingInclusion",
+    "MarkdownChunker",
+    "chunk_markdown_tree",
+    # Streaming
+    "AdaptiveStreamingProcessor",
+    "StreamingMarkdownProcessor",
+    "stream_markdown_file",
+    "stream_markdown_string",
+    # Filters
+    "ChunkQualityConfig",
+    "ChunkQualityFilter",
+    # Enrichment
+    "EnrichedChunkData",
+    "build_enriched_text",
+    "enrich_chunk",
+    "extract_heading_metadata",
+    "format_heading_display",
+    "format_heading_for_display",
+    "get_dynamic_heading_display",
+    "get_relevant_headings_for_display",
+    "is_multiword",
+]

dataknobs_xization/markdown/enrichment.py

@@ -0,0 +1,260 @@
+"""Heading enrichment utilities for RAG-optimized chunk embeddings.
+
+This module provides utilities to enrich chunk content with heading context
+for improved semantic search, while keeping headings out of the displayed content.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+def is_multiword(heading: str) -> bool:
+    """Check if a heading contains multiple words.
+
+    Args:
+        heading: The heading text to check
+
+    Returns:
+        True if the heading has more than one word
+    """
+    return len(heading.split()) > 1
+
+
+def format_heading_display(
+    heading_path: list[str],
+    separator: str = " > ",
+) -> str:
+    """Format a heading path for display.
+
+    Args:
+        heading_path: List of headings from root to chunk
+        separator: Separator to use between headings
+
+    Returns:
+        Formatted heading path string
+    """
+    if not heading_path:
+        return ""
+    return separator.join(heading_path)
+
+
+def get_dynamic_heading_display(
+    heading_path: list[str],
+    content_length: int,
+    small_threshold: int = 200,
+    medium_threshold: int = 800,
+) -> str:
+    """Get heading display based on content length.
+
+    Dynamic heading inclusion:
+    - Small chunks (< small_threshold): Full heading path
+    - Medium chunks (< medium_threshold): Last 2 headings
+    - Large chunks: No headings
+
+    Args:
+        heading_path: List of headings from root to chunk
+        content_length: Length of chunk content in characters
+        small_threshold: Max chars for "small" chunks
+        medium_threshold: Max chars for "medium" chunks
+
+    Returns:
+        Formatted heading display string
+    """
+    if not heading_path:
+        return ""
+
+    if content_length <= small_threshold:
+        # Small chunks: include full heading path
+        return format_heading_display(heading_path)
+    elif content_length <= medium_threshold:
+        # Medium chunks: include last 2 heading levels
+        relevant = heading_path[-2:] if len(heading_path) > 2 else heading_path
+        return format_heading_display(relevant)
+    else:
+        # Large chunks: omit headings
+        return ""
+
+
+@dataclass
+class EnrichedChunkData:
+    """Data for a chunk enriched with heading context.
+
+    Attributes:
+        content: Clean content text (no headings)
+        embedding_text: Text to use for embedding (heading-enriched)
+        heading_path: List of headings from root to chunk
+        heading_display: Formatted heading path for display
+        content_length: Length of clean content in characters
+    """
+
+    content: str
+    embedding_text: str
+    heading_path: list[str]
+    heading_display: str
+    content_length: int
+
+
+def build_enriched_text(heading_path: list[str], content: str) -> str:
+    """Build text for embedding with relevant heading context.
+
+    Uses a modified approach where headings are included up from the chunk
+    until and including the first multi-word heading. This provides semantic
+    context without over-weighting deep, single-word labels like "Example".
+
+    Args:
+        heading_path: List of heading texts from root to chunk
+        content: The chunk content text
+
+    Returns:
+        Enriched text suitable for embedding
+
+    Examples:
+        >>> build_enriched_text(["Patterns", "Chain-of-Thought", "Example"], "code here")
+        'Chain-of-Thought: Example: code here'
+
+        >>> build_enriched_text(["Setup"], "install steps")
+        'Setup: install steps'
+
+        >>> build_enriched_text(["API Reference", "Authentication", "OAuth 2.0"], "...")
+        'Authentication: OAuth 2.0: ...'
+
+        >>> build_enriched_text([], "standalone content")
+        'standalone content'
+    """
+    if not heading_path:
+        return content
+
+    # Walk backwards from deepest heading to find relevant context
+    relevant_headings = []
+    for heading in reversed(heading_path):
+        relevant_headings.insert(0, heading)
+        # Stop after including a multi-word heading
+        if len(heading.split()) > 1:
+            break
+
+    # Build the enriched text
+    if relevant_headings:
+        prefix = ": ".join(relevant_headings)
+        return f"{prefix}: {content}"
+
+    return content
+
+
+def extract_heading_metadata(
+    headings: list[str],
+    heading_levels: list[int],
+    separator: str = " > ",
+) -> dict[str, Any]:
+    """Extract heading metadata for storage.
+
+    Args:
+        headings: List of heading texts from root to chunk
+        heading_levels: Corresponding heading levels (1-6)
+        separator: Separator for display string
+
+    Returns:
+        Dictionary with heading metadata fields
+    """
+    return {
+        "heading_path": headings,
+        "heading_levels": heading_levels,
+        "heading_display": separator.join(headings) if headings else "",
+        "heading_depth": len(headings),
+    }
+
+
+def get_relevant_headings_for_display(
+    heading_path: list[str],
+    content_length: int,
+    small_threshold: int = 200,
+    medium_threshold: int = 800,
+) -> list[str]:
+    """Get headings to display based on content length.
+
+    Implements dynamic heading inclusion:
+    - Small chunks: Full heading path (need context)
+    - Medium chunks: Last 2 heading levels
+    - Large chunks: No headings (content is self-contained)
+
+    Args:
+        heading_path: List of heading texts from root to chunk
+        content_length: Length of chunk content in characters
+        small_threshold: Max chars for "small" chunks
+        medium_threshold: Max chars for "medium" chunks
+
+    Returns:
+        List of headings to display
+    """
+    if not heading_path:
+        return []
+
+    if content_length < small_threshold:
+        # Small chunks: include full heading path
+        return heading_path
+    elif content_length < medium_threshold:
+        # Medium chunks: include last 2 heading levels
+        return heading_path[-2:] if len(heading_path) > 2 else heading_path
+    else:
+        # Large chunks: omit headings
+        return []
+
+
+def format_heading_for_display(
+    headings: list[str],
+    heading_levels: list[int] | None = None,
+    format_style: str = "markdown",
+) -> str:
+    """Format headings for display in LLM context.
+
+    Args:
+        headings: List of heading texts to display
+        heading_levels: Corresponding levels (used for markdown format)
+        format_style: "markdown" for # syntax, "path" for > separated
+
+    Returns:
+        Formatted heading string
+    """
+    if not headings:
+        return ""
+
+    if format_style == "path":
+        return " > ".join(headings)
+
+    if format_style == "markdown" and heading_levels:
+        lines = []
+        for heading, level in zip(headings, heading_levels):
+            lines.append(f"{'#' * level} {heading}")
+        return "\n".join(lines)
+
+    # Default: just join with separator
+    return " > ".join(headings)
+
+
+def enrich_chunk(
+    content: str,
+    headings: list[str],
+    heading_levels: list[int],
+) -> EnrichedChunkData:
+    """Create fully enriched chunk data from raw components.
+
+    Convenience function that combines all enrichment operations.
+
+    Args:
+        content: Raw chunk content text
+        headings: List of heading texts from root to chunk
+        heading_levels: Corresponding heading levels
+
+    Returns:
+        EnrichedChunkData with all computed fields
+    """
+    embedding_text = build_enriched_text(headings, content)
+
+    return EnrichedChunkData(
+        content=content,
+        embedding_text=embedding_text,
+        heading_path=headings,
+        heading_display=" > ".join(headings) if headings else "",
+        content_length=len(content),
+    )

dataknobs_xization/markdown/filters.py

@@ -0,0 +1,236 @@
+"""Quality filters for markdown chunks.
+
+This module provides filtering utilities to identify and remove low-quality
+chunks that would not contribute meaningful content to RAG retrieval.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from dataknobs_xization.markdown.md_chunker import Chunk
+
+
+@dataclass
+class ChunkQualityConfig:
+    """Configuration for chunk quality filtering.
+
+    Attributes:
+        min_content_chars: Minimum characters of non-heading content
+        min_alphanumeric_ratio: Minimum ratio of alphanumeric to total chars
+        skip_heading_only: Skip chunks with only headings (no body content)
+        min_words: Minimum word count for content
+        allow_code_blocks: Allow short code blocks that would otherwise be filtered
+        allow_tables: Allow short tables that would otherwise be filtered
+    """
+
+    min_content_chars: int = 50
+    min_alphanumeric_ratio: float = 0.3
+    skip_heading_only: bool = True
+    min_words: int = 5
+    allow_code_blocks: bool = True
+    allow_tables: bool = True
+
+
+class ChunkQualityFilter:
+    """Filter for identifying and removing low-quality chunks.
+
+    This filter helps ensure that only meaningful content is indexed
+    for RAG retrieval, reducing noise and improving retrieval quality.
+    """
+
+    def __init__(self, config: ChunkQualityConfig | None = None):
+        """Initialize the quality filter.
+
+        Args:
+            config: Quality configuration, uses defaults if not provided
+        """
+        self.config = config or ChunkQualityConfig()
+
+    def is_valid(self, chunk: Chunk) -> bool:
+        """Check if a chunk meets quality thresholds.
+
+        Args:
+            chunk: The chunk to evaluate
+
+        Returns:
+            True if chunk should be kept, False if it should be filtered
+        """
+        # Get node type from custom metadata
+        node_type = chunk.metadata.custom.get("node_type", "body")
+
+        # Special handling for code blocks and tables
+        if node_type == "code" and self.config.allow_code_blocks:
+            return self._is_valid_code_block(chunk)
+        if node_type == "table" and self.config.allow_tables:
+            return self._is_valid_table(chunk)
+
+        # Extract content without heading markers
+        content = self._extract_content_text(chunk.text)
+
+        # Check for heading-only chunks
+        if self.config.skip_heading_only and not content.strip():
+            return False
+
+        # Check minimum content length
+        if len(content) < self.config.min_content_chars:
+            return False
+
+        # Check alphanumeric ratio
+        if not self._meets_alphanumeric_threshold(content):
+            return False
+
+        # Check word count
+        if not self._meets_word_count(content):
+            return False
+
+        return True
+
+    def _extract_content_text(self, text: str) -> str:
+        """Extract content text, removing markdown heading markers.
+
+        Args:
+            text: Raw chunk text
+
+        Returns:
+            Content without heading lines
+        """
+        lines = text.split("\n")
+        content_lines = []
+
+        for line in lines:
+            # Skip markdown heading lines
+            if re.match(r"^#+\s+", line):
+                continue
+            content_lines.append(line)
+
+        return "\n".join(content_lines)
+
+    def _meets_alphanumeric_threshold(self, text: str) -> bool:
+        """Check if text meets minimum alphanumeric ratio.
+
+        Args:
+            text: Text to check
+
+        Returns:
+            True if ratio is met
+        """
+        if not text:
+            return False
+
+        alphanumeric_count = sum(1 for c in text if c.isalnum())
+        total_count = len(text)
+
+        if total_count == 0:
+            return False
+
+        ratio = alphanumeric_count / total_count
+        return ratio >= self.config.min_alphanumeric_ratio
+
+    def _meets_word_count(self, text: str) -> bool:
+        """Check if text meets minimum word count.
+
+        Args:
+            text: Text to check
+
+        Returns:
+            True if word count is met
+        """
+        words = text.split()
+        return len(words) >= self.config.min_words
+
+    def _is_valid_code_block(self, chunk: Chunk) -> bool:
+        """Check if a code block chunk is valid.
+
+        Code blocks are given more lenient filtering since they may be
+        short but still valuable (e.g., single function definitions).
+
+        Args:
+            chunk: Code block chunk
+
+        Returns:
+            True if code block should be kept
+        """
+        # Code blocks must have at least some content
+        content = chunk.text.strip()
+        if not content:
+            return False
+
+        # Allow code blocks with at least one non-whitespace line
+        lines = [line for line in content.split("\n") if line.strip()]
+        return len(lines) >= 1
+
+    def _is_valid_table(self, chunk: Chunk) -> bool:
+        """Check if a table chunk is valid.
+
+        Tables are given more lenient filtering since they may be
+        compact but information-rich.
+
+        Args:
+            chunk: Table chunk
+
+        Returns:
+            True if table should be kept
+        """
+        # Tables must have at least some content
+        content = chunk.text.strip()
+        if not content:
+            return False
+
+        # Tables should have at least header row and one data row
+        lines = [line for line in content.split("\n") if line.strip()]
+        return len(lines) >= 2
+
+    def filter_chunks(self, chunks: list[Chunk]) -> list[Chunk]:
+        """Filter a list of chunks, keeping only valid ones.
+
+        Args:
+            chunks: List of chunks to filter
+
+        Returns:
+            List of chunks that pass quality thresholds
+        """
+        return [chunk for chunk in chunks if self.is_valid(chunk)]
+
+    def get_rejection_reason(self, chunk: Chunk) -> str | None:
+        """Get the reason a chunk would be rejected.
+
+        Useful for debugging and understanding filtering behavior.
+
+        Args:
+            chunk: The chunk to evaluate
+
+        Returns:
+            Rejection reason string, or None if chunk is valid
+        """
+        node_type = chunk.metadata.custom.get("node_type", "body")
+
+        if node_type == "code" and self.config.allow_code_blocks:
+            if not self._is_valid_code_block(chunk):
+                return "Empty code block"
+            return None
+
+        if node_type == "table" and self.config.allow_tables:
+            if not self._is_valid_table(chunk):
+                return "Empty or single-row table"
+            return None
+
+        content = self._extract_content_text(chunk.text)
+
+        if self.config.skip_heading_only and not content.strip():
+            return "Heading-only chunk (no body content)"
+
+        if len(content) < self.config.min_content_chars:
+            return f"Content too short ({len(content)} < {self.config.min_content_chars} chars)"
+
+        if not self._meets_alphanumeric_threshold(content):
+            return f"Alphanumeric ratio below threshold ({self.config.min_alphanumeric_ratio})"
+
+        if not self._meets_word_count(content):
+            words = len(content.split())
+            return f"Word count too low ({words} < {self.config.min_words} words)"
+
+        return None