exonware-xwsystem 0.0.1.411__py3-none-any.whl → 0.1.0.3__py3-none-any.whl

exonware/xwsystem/io/data_operations.py

@@ -16,7 +16,7 @@ lazy, paged, and atomic access features without re-implementing I/O logic.
 Company: eXonware.com
 Author: Eng. Muhammad AlShehri
 Email: connect@exonware.com
-Version: 0.0.1.411
+Version: 0.1.0.3
 Generation Date: 15-Dec-2025
 """
 
@@ -28,16 +28,85 @@ from typing import Any, Callable, Optional
 from abc import ABC, abstractmethod
 import json
 import os
+import platform
+import sys
 import tempfile
+import multiprocessing as mp
+from concurrent.futures import ProcessPoolExecutor, as_completed
 
 from .serialization.auto_serializer import AutoSerializer
-from ..config.logging_setup import get_logger
+from exonware.xwsystem.config.logging_setup import get_logger
+from exonware.xwsystem.config.performance import get_performance_config
 
 
 logger = get_logger(__name__)
 
 
 JsonMatchFn = Callable[[Any], bool]
+
+
+def _process_chunk_worker(args: tuple[int, int, int, str, str, str | None, int | None, bool]) -> tuple[list[int] | None, dict[str, int], int]:
+    """
+    Process a single chunk (runs in worker process).
+    
+    This is a module-level function to make it picklable for multiprocessing.
+    """
+    chunk_id, start_offset, end_offset, file_path_str, encoding, id_field_arg, max_id_index_arg, build_line_offsets_arg = args
+    chunk_line_offsets: list[int] | None = [] if build_line_offsets_arg else None
+    chunk_id_index: dict[str, int] = {}
+    lines_processed = 0
+    
+    # Import parser in worker process (can't pickle serializer)
+    from exonware.xwsystem.io.serialization.parsers.registry import get_best_available_parser
+    parser = get_best_available_parser()
+    
+    try:
+        with open(file_path_str, "rb") as f:
+            f.seek(start_offset)
+            current_offset = start_offset
+            
+            while current_offset < end_offset:
+                line_start = current_offset
+                line = f.readline()
+                
+                if not line:
+                    break
+                
+                current_offset = f.tell()
+                
+                # Skip if we've gone past the end
+                if line_start >= end_offset:
+                    break
+                
+                # Optimize: Check empty lines early (match example code pattern)
+                raw = line.strip()
+                if not raw:
+                    continue
+                
+                # Track line offset if requested, calculate line_idx once
+                if build_line_offsets_arg:
+                    chunk_line_offsets.append(line_start)
+                    line_idx = len(chunk_line_offsets) - 1
+                else:
+                    line_idx = lines_processed
+                
+                if id_field_arg and (max_id_index_arg is None or len(chunk_id_index) < max_id_index_arg):
+                    try:
+                        # Parser accepts bytes directly (hybrid parser handles it)
+                        obj = parser.loads(raw)
+                        if isinstance(obj, dict) and id_field_arg in obj:
+                            id_val = str(obj[id_field_arg])
+                            chunk_id_index[id_val] = line_idx
+                    except Exception:
+                        # Skip invalid lines (fallback indexing)
+                        pass
+                
+                lines_processed += 1
+    except Exception as e:
+        # Can't use logger in worker process, just pass
+        pass
+    
+    return (chunk_line_offsets, chunk_id_index, lines_processed)
 JsonUpdateFn = Callable[[Any], Any]
 
 
@@ -283,7 +352,7 @@ class NDJSONDataOperations(ADataOperations):
                 try:
                     tmp_path.unlink()
                 except OSError:
-                    # Best-effort cleanup; do not mask original error.
+                    # Cleanup attempt; do not mask original error.
                     logger.debug("Failed to cleanup temp file %s", tmp_path)
 
     # ------------------------------------------------------------------
@@ -297,22 +366,75 @@ class NDJSONDataOperations(ADataOperations):
         encoding: str = "utf-8",
         id_field: str | None = None,
         max_id_index: int | None = None,
+        use_parallel: bool | None = None,
+        num_workers: int | None = None,
+        chunk_size_mb: int = 100,
+        build_line_offsets: bool = True,
     ) -> JsonIndex:
         """
         One-time full scan to build an index:
           - line_offsets: byte offset of each JSON line
           - optional id_index: obj[id_field] -> line_number
+        
+        Args:
+            file_path: Path to JSONL file
+            encoding: File encoding (default: utf-8)
+            id_field: Optional field name to build id_index
+            max_id_index: Maximum entries in id_index (None = unlimited)
+            use_parallel: Enable parallel processing (None = auto-detect based on file size)
+            num_workers: Number of worker processes (None = CPU count)
+            chunk_size_mb: Chunk size in MB for parallel processing (default: 100MB)
+            build_line_offsets: If True, build line_offsets list (default: True, set False for faster id_index-only builds)
+        
+        Returns:
+            JsonIndex with line_offsets (if build_line_offsets=True) and optional id_index
         """
         target = Path(file_path)
         if not target.exists():
             raise FileNotFoundError(str(target))
 
-        line_offsets: list[int] = []
+        # Auto-detect parallel based on config
+        perf_config = get_performance_config()
+        if use_parallel is None:
+            if not perf_config.enable_parallel_index:
+                use_parallel = False
+            else:
+                file_size_mb = target.stat().st_size / 1_048_576  # 1024 * 1024
+                use_parallel = file_size_mb > perf_config.parallel_index_threshold_mb
+        
+        # Use config defaults for workers and chunk size
+        if num_workers is None:
+            num_workers = perf_config.parallel_index_workers
+        if chunk_size_mb == 100:  # Only use default if not explicitly set
+            chunk_size_mb = perf_config.parallel_index_chunk_size_mb
+
+        # Use parallel processing if enabled and file is large enough
+        if use_parallel:
+            try:
+                return self._build_index_parallel(
+                    target,
+                    encoding=encoding,
+                    id_field=id_field,
+                    max_id_index=max_id_index,
+                    num_workers=num_workers,
+                    chunk_size_mb=chunk_size_mb,
+                    build_line_offsets=build_line_offsets,
+                )
+            except Exception as e:
+                logger.warning(f"Parallel index building failed, falling back to single-threaded: {e}")
+                # Fall through to single-threaded
+
+        # Single-threaded implementation (optimized - matches example code exactly)
+        line_offsets: list[int] | None = [] if build_line_offsets else None
         id_index: dict[str, int] | None = {} if id_field else None
 
         size = target.stat().st_size
         mtime = target.stat().st_mtime
 
+        # Cache parser instance (matches example code pattern)
+        from exonware.xwsystem.io.serialization.parsers.registry import get_best_available_parser
+        parser = get_best_available_parser()
+
         offset = 0
         with target.open("rb") as f:
             line_no = 0
@@ -320,28 +442,173 @@ class NDJSONDataOperations(ADataOperations):
                 line = f.readline()
                 if not line:
                     break
-                line_offsets.append(offset)
+                if build_line_offsets:
+                    line_offsets.append(offset)
 
                 if id_index is not None:
                     try:
-                        text = line.decode(encoding).strip()
-                        if text:
-                            obj = self._serializer.detect_and_deserialize(
-                                text, file_path=target, format_hint="JSON"
-                            )
+                        # Match example code exactly: strip bytes, parse directly
+                        raw = line.strip()
+                        if raw:
+                            # Parser accepts bytes directly (hybrid parser handles it)
+                            obj = parser.loads(raw)
                             if isinstance(obj, dict) and id_field in obj:
                                 id_val = str(obj[id_field])
                                 if max_id_index is None or len(id_index) < max_id_index:
                                     id_index[id_val] = line_no
                     except Exception:
-                        # Index should be best-effort and robust to bad lines.
-                        logger.debug("Skipping line %s while building id index", line_no)
+                        # Index is robust to bad lines.
+                        # Skip invalid lines silently for performance
+                        pass
 
                 offset += len(line)
                 line_no += 1
 
         meta = JsonIndexMeta(path=str(target), size=size, mtime=mtime, version=1)
         return JsonIndex(meta=meta, line_offsets=line_offsets, id_index=id_index)
+    
+    def _build_index_parallel(
+        self,
+        file_path: Path,
+        *,
+        encoding: str = "utf-8",
+        id_field: str | None = None,
+        max_id_index: int | None = None,
+        num_workers: int | None = None,
+        chunk_size_mb: int = 100,
+        build_line_offsets: bool = True,
+    ) -> JsonIndex:
+        """
+        Parallel index building using multiple CPU cores.
+        
+        This is an internal method called by build_index() when use_parallel=True.
+        """
+        if num_workers is None:
+            # Optimize: Simple formula - 1 worker per 10MB (capped at ProcessPoolExecutor limit)
+            # ProcessPoolExecutor max_workers limit is 61 on Windows
+            file_size_mb = file_path.stat().st_size / 1_048_576  # 1024 * 1024
+            calculated_workers = int(file_size_mb / 10)  # 1 worker per 10MB
+            cpu_count = mp.cpu_count()
+            # Windows has a hard limit of 61 workers for ProcessPoolExecutor
+            # Unix/Linux systems have no such limit
+            # Use Python's native platform module for cross-platform detection
+            if platform.system() == 'Windows':
+                num_workers = max(cpu_count, min(61, calculated_workers))
+            else:
+                num_workers = max(cpu_count, calculated_workers)
+        
+        file_size = file_path.stat().st_size
+        chunk_size_bytes = chunk_size_mb * 1_048_576  # 1024 * 1024
+        
+        # If file is too small, fall back to single-threaded
+        if file_size < chunk_size_bytes * 2:
+            raise ValueError("File too small for parallel processing")
+        
+        # Split file into chunks
+        chunks = []
+        current_offset = 0
+        chunk_id = 0
+        
+        while current_offset < file_size:
+            chunk_end = min(current_offset + chunk_size_bytes, file_size)
+            chunks.append((chunk_id, current_offset, chunk_end))
+            current_offset = chunk_end
+            chunk_id += 1
+        
+        # Limit number of chunks
+        if len(chunks) > num_workers * 2:
+            merged_chunks = []
+            for i in range(0, len(chunks), max(1, len(chunks) // num_workers)):
+                chunk_group = chunks[i:i + max(1, len(chunks) // num_workers)]
+                if chunk_group:
+                    merged_chunks.append((
+                        chunk_group[0][0],
+                        chunk_group[0][1],
+                        chunk_group[-1][2]
+                    ))
+            chunks = merged_chunks
+        
+        logger.debug(f"Processing {len(chunks)} chunks with {num_workers} workers")
+        
+        # Process chunks in parallel
+        line_offsets: list[int] | None = [] if build_line_offsets else None
+        id_index: dict[str, int] | None = {} if id_field else None
+        
+        # Prepare arguments for worker processes
+        chunk_args = [
+            (chunk[0], chunk[1], chunk[2], str(file_path), encoding, id_field, max_id_index, build_line_offsets)
+            for chunk in chunks
+        ]
+        
+        # Execute parallel processing
+        with ProcessPoolExecutor(max_workers=num_workers) as executor:
+            futures = {
+                executor.submit(_process_chunk_worker, args): args[0]
+                for args in chunk_args
+            }
+            
+            # Optimize: Use dict for O(1) lookup instead of sorting
+            chunk_results_dict: dict[int, tuple[list[int], dict[str, int]]] = {}
+            for future in as_completed(futures):
+                try:
+                    chunk_offsets, chunk_ids, _ = future.result()
+                    chunk_id = futures[future]
+                    chunk_results_dict[chunk_id] = (chunk_offsets, chunk_ids)
+                except Exception as e:
+                    logger.warning(f"Chunk processing failed: {e}")
+                    raise
+        
+        # Merge results (process in order by chunk_id)
+        if build_line_offsets:
+            # Optimize: Pre-calculate total size for memory allocation
+            total_offsets = sum(len(offsets) if offsets else 0 for offsets, _ in chunk_results_dict.values())
+            if total_offsets > 0:
+                # Pre-allocate list for performance
+                line_offsets = [0] * total_offsets
+                current_idx = 0
+            else:
+                line_offsets = []
+                current_idx = 0
+        else:
+            current_idx = 0
+        
+        for chunk_id in sorted(chunk_results_dict.keys()):
+            chunk_offsets, chunk_ids = chunk_results_dict[chunk_id]
+            
+            # Merge line_offsets if building them
+            if build_line_offsets and chunk_offsets:
+                # Optimize: Use slice assignment for faster extend
+                if total_offsets > 0:
+                    line_offsets[current_idx:current_idx + len(chunk_offsets)] = chunk_offsets
+                    base_line = current_idx
+                    current_idx += len(chunk_offsets)
+                else:
+                    base_line = len(line_offsets)
+                    line_offsets.extend(chunk_offsets)
+            else:
+                # Calculate base_line for id_index even without line_offsets
+                base_line = current_idx
+                if chunk_offsets:
+                    current_idx += len(chunk_offsets)
+                else:
+                    # Estimate: assume average line size if we don't have offsets
+                    current_idx += 300  # Rough estimate
+            
+            if id_index is not None and chunk_ids:
+                # Optimize: Batch update with dict.update() if no limit
+                if max_id_index is None:
+                    # Fast path: no limit, use dict comprehension + update
+                    id_index.update({id_val: base_line + rel_line for id_val, rel_line in chunk_ids.items()})
+                else:
+                    # Slower path: check limit per item
+                    for id_val, rel_line in chunk_ids.items():
+                        if len(id_index) < max_id_index:
+                            id_index[id_val] = base_line + rel_line
+        
+        size = file_path.stat().st_size
+        mtime = file_path.stat().st_mtime
+        meta = JsonIndexMeta(path=str(file_path), size=size, mtime=mtime, version=1)
+        return JsonIndex(meta=meta, line_offsets=line_offsets, id_index=id_index)
 
     def indexed_get_by_line(
         self,
@@ -476,5 +743,3 @@ __all__ = [
     "ADataOperations",
     "NDJSONDataOperations",
 ]
-
-

exonware/xwsystem/io/defs.py

@@ -1,8 +1,9 @@
+#exonware/xwsystem/src/exonware/xwsystem/io/defs.py
 """
 Company: eXonware.com
 Author: Eng. Muhammad AlShehri
 Email: connect@exonware.com
-Version: 0.0.1.411
+Version: 0.1.0.3
 Generation Date: 30-Oct-2025
 
 IO module definitions - ALL enums and types in ONE place.
@@ -117,7 +118,7 @@ class PagingMode(Enum):
     LINE = "line"                # Page by line counts
     RECORD = "record"            # Page by record boundaries
     SMART = "smart"              # Adaptive paging
-    AUTO = "auto"                # Auto-detect best strategy
+    AUTO = "auto"                # Auto-detect strategy
 
 
 # From file
@@ -221,7 +222,7 @@ class CompressionLevel(Enum):
     """Compression level presets."""
     FAST = 1                # Fastest compression
     BALANCED = 6            # Balance speed/size
-    BEST = 9                # Best compression
+    BEST = 9                # Maximum compression
 
 
 

exonware/xwsystem/io/errors.py

@@ -1,8 +1,9 @@
+#exonware/xwsystem/src/exonware/xwsystem/io/errors.py
 """
 Company: eXonware.com
 Author: Eng. Muhammad AlShehri  
 Email: connect@exonware.com
-Version: 0.0.1.411
+Version: 0.1.0.3
 Generation Date: 30-Oct-2025
 
 IO module errors - ALL exceptions in ONE place.
@@ -10,7 +11,7 @@ IO module errors - ALL exceptions in ONE place.
 Consolidated from all submodules for maintainability.
 """
 
-from typing import Any, Optional, Union
+from typing import Any, Optional
 from pathlib import Path