aiecs 1.0.1__py3-none-any.whl → 1.7.17__py3-none-any.whl

aiecs/application/knowledge_graph/builder/data_reshaping.py

@@ -0,0 +1,293 @@
+"""
+Data Reshaping for Knowledge Graph Import
+
+Provides utilities to convert wide format data to normalized graph structures
+and vice versa, enabling efficient import of datasets with many columns.
+"""
+
+from typing import List, Optional, Dict, Any, Union
+from dataclasses import dataclass
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Check for pandas availability
+try:
+    import pandas as pd
+    PANDAS_AVAILABLE = True
+except ImportError:
+    PANDAS_AVAILABLE = False
+
+
+@dataclass
+class ReshapeResult:
+    """
+    Result of data reshaping operation
+    
+    Attributes:
+        data: Reshaped DataFrame
+        original_shape: Original (rows, cols) shape
+        new_shape: New (rows, cols) shape
+        id_columns: Columns used as identifiers
+        variable_column: Name of variable column (for melt)
+        value_column: Name of value column (for melt)
+        warnings: List of warnings
+    """
+    data: 'pd.DataFrame'
+    original_shape: tuple
+    new_shape: tuple
+    id_columns: List[str]
+    variable_column: Optional[str] = None
+    value_column: Optional[str] = None
+    warnings: List[str] = None
+    
+    def __post_init__(self):
+        if self.warnings is None:
+            self.warnings = []
+
+
+class DataReshaping:
+    """
+    Utility class for reshaping structured data
+    
+    Provides methods to convert between wide and long formats,
+    enabling normalized graph structures from wide datasets.
+    """
+    
+    @staticmethod
+    def melt(
+        df: 'pd.DataFrame',
+        id_vars: List[str],
+        value_vars: Optional[List[str]] = None,
+        var_name: str = 'variable',
+        value_name: str = 'value',
+        dropna: bool = True,
+    ) -> ReshapeResult:
+        """
+        Convert wide format to long format (melt operation)
+        
+        Transforms data from wide format (many columns) to long format
+        (fewer columns, more rows), which is ideal for normalized graph structures.
+        
+        Args:
+            df: DataFrame to reshape
+            id_vars: Columns to use as identifier variables
+            value_vars: Columns to unpivot (default: all columns except id_vars)
+            var_name: Name for the variable column (default: 'variable')
+            value_name: Name for the value column (default: 'value')
+            dropna: Whether to drop rows with missing values (default: True)
+        
+        Returns:
+            ReshapeResult with reshaped data and metadata
+        
+        Example:
+            ```python
+            # Wide format: sample_id, option1, option2, option3
+            # Long format: sample_id, variable, value
+            
+            result = DataReshaping.melt(
+                df,
+                id_vars=['sample_id'],
+                value_vars=['option1', 'option2', 'option3'],
+                var_name='option_name',
+                value_name='option_value'
+            )
+            ```
+        """
+        if not PANDAS_AVAILABLE:
+            raise ImportError("pandas is required for data reshaping")
+        
+        original_shape = df.shape
+        warnings = []
+        
+        # If value_vars not specified, use all columns except id_vars
+        if value_vars is None:
+            value_vars = [col for col in df.columns if col not in id_vars]
+            warnings.append(f"Auto-detected {len(value_vars)} value columns")
+        
+        # Perform melt operation
+        melted = pd.melt(
+            df,
+            id_vars=id_vars,
+            value_vars=value_vars,
+            var_name=var_name,
+            value_name=value_name,
+        )
+        
+        # Drop NA values if requested
+        if dropna:
+            rows_before = len(melted)
+            melted = melted.dropna(subset=[value_name])
+            rows_dropped = rows_before - len(melted)
+            if rows_dropped > 0:
+                warnings.append(f"Dropped {rows_dropped} rows with missing values")
+        
+        new_shape = melted.shape
+        
+        return ReshapeResult(
+            data=melted,
+            original_shape=original_shape,
+            new_shape=new_shape,
+            id_columns=id_vars,
+            variable_column=var_name,
+            value_column=value_name,
+            warnings=warnings,
+        )
+
+    @staticmethod
+    def pivot(
+        df: 'pd.DataFrame',
+        index: Union[str, List[str]],
+        columns: str,
+        values: str,
+        aggfunc: str = 'first',
+        fill_value: Optional[Any] = None,
+    ) -> ReshapeResult:
+        """
+        Convert long format to wide format (pivot operation)
+
+        Transforms data from long format to wide format, creating columns
+        from unique values in the specified column.
+
+        Args:
+            df: DataFrame to reshape
+            index: Column(s) to use as index (identifier)
+            columns: Column whose unique values become new columns
+            values: Column containing values to populate the new columns
+            aggfunc: Aggregation function if multiple values per group (default: 'first')
+            fill_value: Value to use for missing data (default: None)
+
+        Returns:
+            ReshapeResult with reshaped data and metadata
+
+        Example:
+            ```python
+            # Long format: sample_id, option_name, option_value
+            # Wide format: sample_id, option1, option2, option3
+
+            result = DataReshaping.pivot(
+                df,
+                index='sample_id',
+                columns='option_name',
+                values='option_value'
+            )
+            ```
+        """
+        if not PANDAS_AVAILABLE:
+            raise ImportError("pandas is required for data reshaping")
+
+        original_shape = df.shape
+        warnings = []
+
+        # Perform pivot operation
+        try:
+            pivoted = df.pivot_table(
+                index=index,
+                columns=columns,
+                values=values,
+                aggfunc=aggfunc,
+                fill_value=fill_value,
+            )
+
+            # Reset index to make it a regular column
+            pivoted = pivoted.reset_index()
+
+            # Flatten column names if multi-level
+            if isinstance(pivoted.columns, pd.MultiIndex):
+                pivoted.columns = ['_'.join(map(str, col)).strip('_') for col in pivoted.columns.values]
+                warnings.append("Flattened multi-level column names")
+
+        except Exception as e:
+            raise ValueError(f"Pivot operation failed: {e}")
+
+        new_shape = pivoted.shape
+
+        # Determine id_columns
+        if isinstance(index, str):
+            id_columns = [index]
+        else:
+            id_columns = list(index)
+
+        return ReshapeResult(
+            data=pivoted,
+            original_shape=original_shape,
+            new_shape=new_shape,
+            id_columns=id_columns,
+            warnings=warnings,
+        )
+
+    @staticmethod
+    def detect_wide_format(
+        df: 'pd.DataFrame',
+        threshold_columns: int = 50,
+    ) -> bool:
+        """
+        Detect if DataFrame is in wide format
+
+        Wide format is characterized by many columns relative to rows.
+
+        Args:
+            df: DataFrame to analyze
+            threshold_columns: Minimum number of columns to consider wide (default: 50)
+
+        Returns:
+            True if DataFrame appears to be in wide format
+        """
+        if not PANDAS_AVAILABLE:
+            return False
+
+        num_cols = len(df.columns)
+        num_rows = len(df)
+
+        # Wide format indicators:
+        # 1. Many columns (>= threshold)
+        # 2. More columns than rows (or close to it) AND at least 20 columns
+        is_wide = num_cols >= threshold_columns or (num_cols >= 20 and num_cols > num_rows * 0.5)
+
+        return is_wide
+
+    @staticmethod
+    def suggest_melt_config(
+        df: 'pd.DataFrame',
+        id_column_patterns: Optional[List[str]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Suggest melt configuration for wide format data
+
+        Analyzes DataFrame structure to suggest appropriate id_vars and value_vars.
+
+        Args:
+            df: DataFrame to analyze
+            id_column_patterns: Patterns to identify ID columns (default: ['id', 'key', 'name'])
+
+        Returns:
+            Dictionary with suggested melt configuration
+        """
+        if not PANDAS_AVAILABLE:
+            raise ImportError("pandas is required for data reshaping")
+
+        if id_column_patterns is None:
+            id_column_patterns = ['id', 'key', 'name', 'sample', 'subject']
+
+        # Identify potential ID columns
+        id_vars = []
+        for col in df.columns:
+            col_lower = col.lower()
+            if any(pattern in col_lower for pattern in id_column_patterns):
+                id_vars.append(col)
+
+        # If no ID columns found, use first column
+        if not id_vars and len(df.columns) > 0:
+            id_vars = [df.columns[0]]
+
+        # Value columns are all other columns
+        value_vars = [col for col in df.columns if col not in id_vars]
+
+        return {
+            'id_vars': id_vars,
+            'value_vars': value_vars,
+            'var_name': 'variable',
+            'value_name': 'value',
+            'confidence': 0.8 if id_vars else 0.5,
+        }
+

aiecs/application/knowledge_graph/builder/document_builder.py

@@ -0,0 +1,369 @@
+"""
+Document Graph Builder
+
+Builds knowledge graphs from documents (PDF, DOCX, TXT, etc.).
+"""
+
+import asyncio
+from pathlib import Path
+from typing import List, Optional, Dict, Any, Union
+from dataclasses import dataclass, field
+
+from aiecs.application.knowledge_graph.builder.graph_builder import (
+    GraphBuilder,
+    BuildResult,
+)
+from aiecs.application.knowledge_graph.builder.text_chunker import TextChunker
+from aiecs.tools.docs.document_parser_tool import (
+    DocumentParserTool,
+    ParsingStrategy,
+    OutputFormat,
+)
+
+
+@dataclass
+class DocumentBuildResult:
+    """
+    Result of document-to-graph build operation
+
+    Extends BuildResult with document-specific information.
+    """
+
+    document_path: str
+    document_type: str
+    total_chunks: int = 0
+    chunks_processed: int = 0
+    chunk_results: List[BuildResult] = field(default_factory=list)
+    success: bool = True
+    errors: List[str] = field(default_factory=list)
+
+    @property
+    def total_entities_added(self) -> int:
+        """Total entities added across all chunks"""
+        return sum(r.entities_added for r in self.chunk_results)
+
+    @property
+    def total_relations_added(self) -> int:
+        """Total relations added across all chunks"""
+        return sum(r.relations_added for r in self.chunk_results)
+
+
+class DocumentGraphBuilder:
+    """
+    Build knowledge graphs from documents
+
+    Supports multiple document formats:
+    - PDF
+    - DOCX (Microsoft Word)
+    - TXT (Plain text)
+    - And more via AIECS DocumentParserTool
+
+    For large documents, automatically chunks text into manageable pieces.
+
+    Example:
+        ```python
+        builder = DocumentGraphBuilder(
+            graph_builder=graph_builder,
+            chunk_size=1000
+        )
+
+        result = await builder.build_from_document("research_paper.pdf")
+
+        print(f"Processed {result.total_chunks} chunks")
+        print(f"Added {result.total_entities_added} entities")
+        print(f"Added {result.total_relations_added} relations")
+        ```
+    """
+
+    def __init__(
+        self,
+        graph_builder: GraphBuilder,
+        chunk_size: int = 2000,
+        chunk_overlap: int = 200,
+        enable_chunking: bool = True,
+        parallel_chunks: bool = True,
+        max_parallel_chunks: int = 3,
+    ):
+        """
+        Initialize document graph builder
+
+        Args:
+            graph_builder: GraphBuilder instance for text processing
+            chunk_size: Size of text chunks (in characters)
+            chunk_overlap: Overlap between chunks
+            enable_chunking: Whether to chunk large documents
+            parallel_chunks: Process chunks in parallel
+            max_parallel_chunks: Maximum parallel chunk processing
+        """
+        self.graph_builder = graph_builder
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.enable_chunking = enable_chunking
+        self.parallel_chunks = parallel_chunks
+        self.max_parallel_chunks = max_parallel_chunks
+
+        # Initialize document parser (will read config from environment
+        # variables)
+        self.document_parser = DocumentParserTool()
+
+        # Initialize text chunker
+        self.text_chunker = TextChunker(
+            chunk_size=chunk_size,
+            overlap=chunk_overlap,
+            respect_sentences=True,
+        )
+
+    async def build_from_document(
+        self,
+        document_path: Union[str, Path],
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> DocumentBuildResult:
+        """
+        Build knowledge graph from a document
+
+        Args:
+            document_path: Path to document file
+            metadata: Optional metadata to attach to extracted entities/relations
+
+        Returns:
+            DocumentBuildResult with statistics
+        """
+        document_path = str(document_path)
+        result = DocumentBuildResult(document_path=document_path, document_type="unknown")
+
+        try:
+            # Step 1: Parse document to text
+            text = await self._parse_document(document_path)
+
+            if not text or not text.strip():
+                result.success = False
+                result.errors.append("Document parsing returned empty text")
+                return result
+
+            # Determine document type
+            result.document_type = Path(document_path).suffix[1:].lower()  # Remove leading dot
+
+            # Step 2: Chunk text if needed
+            if self.enable_chunking and len(text) > self.chunk_size:
+                chunks = self.text_chunker.chunk_text(text, metadata={"document": document_path})
+                result.total_chunks = len(chunks)
+            else:
+                # Single chunk (small document)
+                from aiecs.application.knowledge_graph.builder.text_chunker import (
+                    TextChunk,
+                )
+
+                chunks = [
+                    TextChunk(
+                        text=text,
+                        start_char=0,
+                        end_char=len(text),
+                        chunk_index=0,
+                        metadata={"document": document_path},
+                    )
+                ]
+                result.total_chunks = 1
+
+            # Step 3: Process each chunk
+            if self.parallel_chunks and len(chunks) > 1:
+                # Process chunks in parallel
+                chunk_results = await self._process_chunks_parallel(chunks, document_path, metadata)
+            else:
+                # Process chunks sequentially
+                chunk_results = await self._process_chunks_sequential(chunks, document_path, metadata)
+
+            result.chunk_results = chunk_results
+            result.chunks_processed = len(chunk_results)
+
+            # Check if all chunks succeeded
+            failed_chunks = [r for r in chunk_results if not r.success]
+            if failed_chunks:
+                result.errors.append(f"{len(failed_chunks)} chunks failed processing")
+
+            result.success = len(failed_chunks) < len(chunks)  # At least some chunks succeeded
+
+        except Exception as e:
+            result.success = False
+            result.errors.append(f"Document processing failed: {str(e)}")
+
+        return result
+
+    async def build_from_documents(
+        self,
+        document_paths: List[Union[str, Path]],
+        parallel: bool = True,
+        max_parallel: int = 3,
+    ) -> List[DocumentBuildResult]:
+        """
+        Build knowledge graph from multiple documents
+
+        Args:
+            document_paths: List of document paths
+            parallel: Process documents in parallel
+            max_parallel: Maximum parallel documents
+
+        Returns:
+            List of DocumentBuildResult objects
+        """
+        if parallel:
+            semaphore = asyncio.Semaphore(max_parallel)
+
+            async def process_one(doc_path):
+                async with semaphore:
+                    return await self.build_from_document(doc_path)
+
+            tasks = [process_one(doc_path) for doc_path in document_paths]
+            gather_results = await asyncio.gather(*tasks, return_exceptions=True)
+
+            # Handle exceptions - convert all to DocumentBuildResult
+            results: List[DocumentBuildResult] = []
+            for i, result in enumerate(gather_results):
+                if isinstance(result, Exception):
+                    error_result = DocumentBuildResult(
+                        document_path=str(document_paths[i]),
+                        document_type="unknown",
+                        success=False,
+                    )
+                    error_result.errors.append(str(result))
+                    results.append(error_result)
+                elif isinstance(result, DocumentBuildResult):
+                    results.append(result)
+                else:
+                    # Fallback for unexpected types
+                    error_result = DocumentBuildResult(
+                        document_path=str(document_paths[i]),
+                        document_type="unknown",
+                        success=False,
+                    )
+                    error_result.errors.append(f"Unexpected result type: {type(result)}")
+                    results.append(error_result)
+
+            return results
+        else:
+            # Sequential processing
+            results = []
+            for doc_path in document_paths:
+                result = await self.build_from_document(doc_path)
+                results.append(result)
+            return results
+
+    async def _parse_document(self, document_path: str) -> str:
+        """
+        Parse document to text using AIECS document parser
+
+        Args:
+            document_path: Path to document
+
+        Returns:
+            Extracted text content
+        """
+        try:
+            # Use document parser tool
+            parse_result = self.document_parser.parse_document(
+                source=document_path,
+                strategy=ParsingStrategy.TEXT_ONLY,
+                output_format=OutputFormat.TEXT,
+            )
+
+            if isinstance(parse_result, dict):
+                return parse_result.get("content", "")
+            elif isinstance(parse_result, str):
+                return parse_result
+            else:
+                return ""
+
+        except Exception:
+            # Fallback: try reading as plain text
+            try:
+                with open(document_path, "r", encoding="utf-8") as f:
+                    return f.read()
+            except Exception as fallback_error:
+                raise RuntimeError(f"Failed to parse document: {str(fallback_error)}")
+
+    async def _process_chunks_parallel(
+        self,
+        chunks: List,
+        document_path: str,
+        metadata: Optional[Dict[str, Any]],
+    ) -> List[BuildResult]:
+        """
+        Process chunks in parallel
+
+        Args:
+            chunks: List of TextChunk objects
+            document_path: Source document path
+            metadata: Optional metadata
+
+        Returns:
+            List of BuildResult objects
+        """
+        semaphore = asyncio.Semaphore(self.max_parallel_chunks)
+
+        async def process_chunk(chunk):
+            async with semaphore:
+                chunk_metadata = {
+                    "document": document_path,
+                    "chunk_index": chunk.chunk_index,
+                    "chunk_start": chunk.start_char,
+                    "chunk_end": chunk.end_char,
+                }
+                if metadata:
+                    chunk_metadata.update(metadata)
+
+                source = f"{document_path}#chunk{chunk.chunk_index}"
+                return await self.graph_builder.build_from_text(text=chunk.text, source=source, metadata=chunk_metadata)
+
+        tasks = [process_chunk(chunk) for chunk in chunks]
+        gather_results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Handle exceptions - convert all to BuildResult
+        results: List[BuildResult] = []
+        for i, result in enumerate(gather_results):
+            if isinstance(result, Exception):
+                error_result = BuildResult(success=False)
+                error_result.errors.append(f"Chunk {i} failed: {str(result)}")
+                results.append(error_result)
+            elif isinstance(result, BuildResult):
+                results.append(result)
+            else:
+                # Fallback for unexpected types
+                error_result = BuildResult(success=False)
+                error_result.errors.append(f"Unexpected result type: {type(result)}")
+                results.append(error_result)
+
+        return results
+
+    async def _process_chunks_sequential(
+        self,
+        chunks: List,
+        document_path: str,
+        metadata: Optional[Dict[str, Any]],
+    ) -> List[BuildResult]:
+        """
+        Process chunks sequentially
+
+        Args:
+            chunks: List of TextChunk objects
+            document_path: Source document path
+            metadata: Optional metadata
+
+        Returns:
+            List of BuildResult objects
+        """
+        results = []
+
+        for chunk in chunks:
+            chunk_metadata = {
+                "document": document_path,
+                "chunk_index": chunk.chunk_index,
+                "chunk_start": chunk.start_char,
+                "chunk_end": chunk.end_char,
+            }
+            if metadata:
+                chunk_metadata.update(metadata)
+
+            source = f"{document_path}#chunk{chunk.chunk_index}"
+            result = await self.graph_builder.build_from_text(text=chunk.text, source=source, metadata=chunk_metadata)
+            results.append(result)
+
+        return results