exonware-xwsystem 0.0.1.410__py3-none-any.whl → 0.1.0.1__py3-none-any.whl

exonware/xwsystem/io/serialization/formats/text/jsonlines.py

@@ -4,7 +4,7 @@
 Company: eXonware.com
 Author: Eng. Muhammad AlShehri
 Email: connect@exonware.com
-Version: 0.0.1.410
+Version: 0.1.0.1
 Generation Date: 02-Nov-2025
 
 JSON Lines (JSONL/NDJSON) Serialization - Newline-Delimited JSON
@@ -26,11 +26,18 @@ from typing import Any, Optional, Union
 from pathlib import Path
 import json
 
-from ...base import ASerialization
-from ...contracts import ISerialization
+from .json import JsonSerializer
+from ...parsers.registry import get_parser
+from ...parsers.base import IJsonParser
+from ....errors import SerializationError
+from ....common.atomic import AtomicFileWriter
+from exonware.xwsystem.config.logging_setup import get_logger
+from exonware.xwsystem.config.performance import get_performance_config
 
+logger = get_logger(__name__)
 
-class JsonLinesSerializer(ASerialization):
+
+class JsonLinesSerializer(JsonSerializer):
     """
     JSON Lines (JSONL/NDJSON) serializer for streaming data.
     
@@ -39,9 +46,16 @@ class JsonLinesSerializer(ASerialization):
     Concrete: JsonLinesSerializer
     """
     
-    def __init__(self):
-        """Initialize JSON Lines serializer."""
-        super().__init__()
+    def __init__(self, parser_name: Optional[str] = None):
+        """
+        Initialize JSON Lines serializer with optional parser selection.
+        
+        Args:
+            parser_name: Parser name ("standard", "orjson", or None for auto-detect)
+        """
+        super().__init__(parser_name=parser_name)
+        # Get parser instance for direct use in line-by-line operations
+        self._parser: IJsonParser = get_parser(parser_name)
     
     @property
     def codec_id(self) -> str:
@@ -67,8 +81,36 @@ class JsonLinesSerializer(ASerialization):
     def codec_types(self) -> list[str]:
         """JSON Lines is a data exchange format."""
         return ["data", "serialization"]
-    
-    def encode(self, data: Any, options: Optional[dict[str, Any]] = None) -> str:
+
+    # -------------------------------------------------------------------------
+    # RECORD / STREAMING CAPABILITIES
+    # -------------------------------------------------------------------------
+
+    @property
+    def supports_record_streaming(self) -> bool:
+        """
+        JSONL is explicitly designed for record-level streaming.
+
+        This enables stream_read_record / stream_update_record to operate in a
+        true streaming fashion (line-by-line) without loading the entire file.
+        """
+        return True
+
+    @property
+    def supports_record_paging(self) -> bool:
+        """
+        JSONL supports efficient record-level paging.
+
+        Paging is implemented as a lightweight line counter that only parses
+        the requested slice of records.
+        """
+        return True
+
+    # -------------------------------------------------------------------------
+    # CORE ENCODE / DECODE
+    # -------------------------------------------------------------------------
+
+    def encode(self, data: Any, *, options: Optional[dict[str, Any]] = None) -> str:
         """
         Encode data to JSON Lines string.
         
@@ -82,14 +124,22 @@ class JsonLinesSerializer(ASerialization):
         if not isinstance(data, list):
             # Single object - wrap in list
             data = [data]
-        
-        lines = []
+
+        opts = options or {}
+        ensure_ascii = opts.get("ensure_ascii", False)
+
+        lines: list[str] = []
         for item in data:
-            lines.append(json.dumps(item, ensure_ascii=False))
-        
-        return '\n'.join(lines)
-    
-    def decode(self, data: Union[str, bytes], options: Optional[dict[str, Any]] = None) -> list[Any]:
+            # Use pluggable parser
+            result = self._parser.dumps(item, ensure_ascii=ensure_ascii)
+            # Convert bytes to str if needed
+            if isinstance(result, bytes):
+                result = result.decode("utf-8")
+            lines.append(result)
+
+        return "\n".join(lines)
+
+    def decode(self, data: Union[str, bytes], *, options: Optional[dict[str, Any]] = None) -> list[Any]:
         """
         Decode JSON Lines string to list of Python objects.
         
@@ -101,16 +151,260 @@ class JsonLinesSerializer(ASerialization):
             List of decoded Python objects
         """
         if isinstance(data, bytes):
-            data = data.decode('utf-8')
-        
+            data = data.decode("utf-8")
+
         # Split by newlines and parse each line
-        lines = data.strip().split('\n')
-        results = []
-        
+        lines = data.strip().split("\n")
+        results: list[Any] = []
+
         for line in lines:
             line = line.strip()
             if line:  # Skip empty lines
-                results.append(json.loads(line))
+                # Use pluggable parser
+                results.append(self._parser.loads(line))
+
+        return results
+
+    # -------------------------------------------------------------------------
+    # RECORD-LEVEL OPERATIONS (True streaming, line-by-line)
+    # -------------------------------------------------------------------------
+
+    def stream_read_record(
+        self,
+        file_path: Union[str, Path],
+        match: callable,
+        projection: Optional[list[Any]] = None,
+        **options: Any,
+    ) -> Any:
+        """
+        Stream-style read of a single logical record from a JSONL file.
+
+        Reads the file line-by-line, parsing each JSON object and returning the
+        first record that satisfies match(record). Optional projection is
+        applied using the base helper to avoid duplicating logic.
+        """
+        path = Path(file_path)
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+
+        # Line-by-line scan – no full-file load
+        with path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                # Use pluggable parser
+                record = self._parser.loads(line)
+                if match(record):
+                    return self._apply_projection(record, projection)
+
+        raise KeyError("No matching record found")
+
+    def stream_update_record(
+        self,
+        file_path: Union[str, Path],
+        match: callable,
+        updater: callable,
+        *,
+        atomic: bool = True,
+        **options: Any,
+    ) -> int:
+        """
+        Stream-style update of logical records in a JSONL file.
+
+        Implementation uses a temp file + AtomicFileWriter pattern to ensure
+        atomicity when atomic=True. Records are processed line-by-line and only
+        the matching records are materialized and updated.
         
+        Supports append-only log optimization for large files (use_append_log=True).
+        """
+        path = Path(file_path)
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+
+        # Check if append-only log should be used
+        perf_config = get_performance_config()
+        use_append_log = options.get("use_append_log", None)
+        if use_append_log is None:
+            if not perf_config.enable_append_log:
+                use_append_log = False
+            else:
+                # Auto-detect: use for files above threshold
+                file_size_mb = path.stat().st_size / (1024 * 1024)
+                use_append_log = file_size_mb > perf_config.append_log_threshold_mb
+
+        # Try append-only log if enabled
+        if use_append_log:
+            try:
+                from .append_only_log import AppendOnlyLog
+                log = AppendOnlyLog(path)
+                
+                # For append-only log, we need to find matching records first
+                # and apply updates, then append to log
+                # This is a simplified version - full implementation would
+                # integrate with index for O(1) lookups
+                updated = 0
+                with path.open("r", encoding="utf-8") as src:
+                    for line in src:
+                        raw = line.rstrip("\n")
+                        if not raw.strip():
+                            continue
+                        
+                        try:
+                            record = self._parser.loads(raw)
+                            if match(record):
+                                # Apply updater
+                                updated_record = updater(record)
+                                
+                                # Extract type and id for log entry
+                                type_name = record.get("@type") or record.get("type") or "Record"
+                                id_value = str(record.get("id", ""))
+                                
+                                # Append to log
+                                log.update_record(type_name, id_value, lambda x: updated_record)
+                                updated += 1
+                        except Exception:
+                            continue
+                
+                return updated
+            except Exception as e:
+                # Fall back to full rewrite if append-only log fails
+                logger.debug(f"Append-only log failed, falling back to full rewrite: {e}")
+
+        # Original full-rewrite implementation
+        updated = 0
+        backup = options.get("backup", True)
+        ensure_ascii = options.get("ensure_ascii", False)
+
+        try:
+            if atomic:
+                # Atomic path: use AtomicFileWriter for temp+replace semantics
+                with AtomicFileWriter(path, backup=backup) as writer:
+                    with path.open("r", encoding="utf-8") as src:
+                        for line in src:
+                            raw = line.rstrip("\n")
+                            if not raw.strip():
+                                # Preserve structural empty lines
+                                writer.write("\n")
+                                continue
+
+                            # Use pluggable parser
+                            record = self._parser.loads(raw)
+                            if match(record):
+                                record = updater(record)
+                                updated += 1
+
+                            # Use pluggable parser for serialization
+                            result = self._parser.dumps(record, ensure_ascii=ensure_ascii)
+                            if isinstance(result, bytes):
+                                result = result.decode("utf-8")
+                            out_line = result + "\n"
+                            writer.write(out_line)
+            else:
+                # Non-atomic fallback: read + rewrite line-by-line
+                new_lines: list[str] = []
+                with path.open("r", encoding="utf-8") as src:
+                    for line in src:
+                        raw = line.rstrip("\n")
+                        if not raw.strip():
+                            new_lines.append("\n")
+                            continue
+
+                        # Use pluggable parser
+                        record = self._parser.loads(raw)
+                        if match(record):
+                            record = updater(record)
+                            updated += 1
+
+                        # Use pluggable parser for serialization
+                        result = self._parser.dumps(record, ensure_ascii=ensure_ascii)
+                        if isinstance(result, bytes):
+                            result = result.decode("utf-8")
+                        new_lines.append(result + "\n")
+
+                path.parent.mkdir(parents=True, exist_ok=True)
+                path.write_text("".join(new_lines), encoding="utf-8")
+
+            return updated
+        except Exception as e:
+            raise SerializationError(
+                f"Failed to stream-update JSONL records in {path}: {e}",
+                format_name=self.format_name,
+                original_error=e,
+            ) from e
+
+    def get_record_page(
+        self,
+        file_path: Union[str, Path],
+        page_number: int,
+        page_size: int,
+        **options: Any,
+    ) -> list[Any]:
+        """
+        Retrieve a logical page of records from a JSONL file.
+
+        Pages are computed by counting logical records (non-empty lines). Only
+        the requested slice is parsed and returned, keeping memory usage
+        proportional to page_size rather than file size.
+        """
+        if page_number < 1 or page_size <= 0:
+            raise ValueError("Invalid page_number or page_size")
+
+        path = Path(file_path)
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+
+        start_index = (page_number - 1) * page_size
+        end_index = start_index + page_size
+
+        results: list[Any] = []
+        current_index = 0
+
+        with path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+
+                if current_index >= end_index:
+                    break
+
+                if current_index >= start_index:
+                    # Use pluggable parser
+                    results.append(self._parser.loads(line))
+
+                current_index += 1
+
         return results
 
+    def get_record_by_id(
+        self,
+        file_path: Union[str, Path],
+        id_value: Any,
+        *,
+        id_field: str = "id",
+        **options: Any,
+    ) -> Any:
+        """
+        Retrieve a logical record by identifier from a JSONL file.
+
+        Performs a streaming linear scan over records, returning the first
+        record where record[id_field] == id_value.
+        """
+        path = Path(file_path)
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+
+        with path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+
+                # Use pluggable parser
+                record = self._parser.loads(line)
+                if isinstance(record, dict) and record.get(id_field) == id_value:
+                    return record
+
+        raise KeyError(f"Record with {id_field}={id_value!r} not found")
+

exonware/xwsystem/io/serialization/formats/text/multipart.py

@@ -2,7 +2,7 @@
 Company: eXonware.com
 Author: Eng. Muhammad AlShehri
 Email: connect@exonware.com
-Version: 0.0.1.410
+Version: 0.1.0.1
 Generation Date: November 2, 2025
 
 Multipart serialization - Multipart form data format.

exonware/xwsystem/io/serialization/formats/text/toml.py

@@ -2,7 +2,7 @@
 Company: eXonware.com
 Author: Eng. Muhammad AlShehri
 Email: connect@exonware.com
-Version: 0.0.1.410
+Version: 0.1.0.1
 Generation Date: November 2, 2025
 
 TOML serialization - Configuration file format.
@@ -180,8 +180,17 @@ class TomlSerializer(ASerialization):
         """
         try:
             if not isinstance(value, dict):
-                raise TypeError("TOML can only serialize dictionaries")
-            
+                # TOML requires a table (dict) at the top level. For data-oriented
+                # use cases (e.g. record lists), transparently wrap common patterns
+                # so that higher-level APIs (record paging, etc.) can still work
+                # uniformly across formats.
+                if isinstance(value, list):
+                    # Auto-wrap top-level list into "items" table.
+                    value = {"items": value}
+                else:
+                    # Fallback: wrap primitive/other types into a single "value" key.
+                    value = {"value": value}
+
             opts = options or {}
             
             # Root cause fixed: Remove None values before encoding (TOML doesn't support None).
@@ -227,6 +236,13 @@ class TomlSerializer(ASerialization):
             
             # Decode from TOML string
             data = tomllib.loads(repr)
+
+            # If this looks like an auto-wrapped list payload (see encode),
+            # unwrap it for callers so that higher-level APIs (including the
+            # generic record-level operations in ASerialization) see the
+            # natural Python structure (a list of records).
+            if isinstance(data, dict) and set(data.keys()) == {"items"} and isinstance(data["items"], list):
+                return data["items"]
             
             return data
             

exonware/xwsystem/io/serialization/formats/text/xml.py

@@ -2,7 +2,7 @@
 Company: eXonware.com
 Author: Eng. Muhammad AlShehri
 Email: connect@exonware.com
-Version: 0.0.1.410
+Version: 0.1.0.1
 Generation Date: November 2, 2025
 
 XML serialization - Extensible Markup Language.
@@ -427,6 +427,13 @@ class XmlSerializer(ASerialization):
         if isinstance(repr, bytes):
             repr = repr.decode('utf-8')
         
+        # Trim leading BOM/whitespace before XML declaration.
+        # Root cause: Some producers emit a blank line or BOM before '<?xml ...?>',
+        # which causes ExpatError: "XML or text declaration not at start of entity".
+        # Priority #2 (Usability): Be forgiving on harmless leading whitespace/BOM
+        # while keeping strict parsing for the actual XML content.
+        repr = repr.lstrip("\ufeff\r\n\t ")
+        
         opts = options or {}
         root_name = opts.get('root', 'root')
         preserve_types = opts.get('preserve_types', False)

exonware/xwsystem/io/serialization/formats/text/yaml.py

@@ -2,7 +2,7 @@
 Company: eXonware.com
 Author: Eng. Muhammad AlShehri
 Email: connect@exonware.com
-Version: 0.0.1.410
+Version: 0.1.0.1
 Generation Date: November 2, 2025
 
 YAML serialization - Human-readable data serialization format.
@@ -13,7 +13,7 @@ Following I→A pattern:
 - Concrete: YamlSerializer
 """
 
-from typing import Any, Optional, Union
+from typing import Any, Optional, Union, Iterator
 from pathlib import Path
 
 from ...base import ASerialization
@@ -88,6 +88,10 @@ class YamlSerializer(ASerialization):
     def supports_streaming(self) -> bool:
         return True  # YAML supports multiple documents
     
+    @property
+    def supports_incremental_streaming(self) -> bool:
+        return True  # YAML supports multi-document streaming
+    
     @property
     def capabilities(self) -> CodecCapability:
         return CodecCapability.BIDIRECTIONAL
@@ -178,4 +182,50 @@ class YamlSerializer(ASerialization):
                 format_name=self.format_name,
                 original_error=e
             )
+    
+    # ========================================================================
+    # INCREMENTAL STREAMING
+    # ========================================================================
+    
+    def incremental_load(
+        self,
+        file_path: Union[str, Path],
+        **options: Any,
+    ) -> Iterator[Any]:
+        """
+        Stream YAML documents one at a time (supports multi-document YAML).
+        
+        Uses PyYAML's safe_load_all() for true streaming without loading
+        entire file into memory.
+        
+        Args:
+            file_path: Path to the YAML file
+            **options: YAML options (Loader, etc.)
+            
+        Yields:
+            Each document from the YAML file one at a time
+            
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            SerializationError: If parsing fails
+        """
+        path = Path(file_path)
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+        
+        opts = options or {}
+        loader = opts.get('Loader', yaml.SafeLoader)
+        
+        try:
+            with path.open("r", encoding="utf-8") as f:
+                # Use safe_load_all for multi-document streaming
+                for document in yaml.safe_load_all(f):
+                    if document is not None:  # Skip empty documents
+                        yield document
+        except (yaml.YAMLError, UnicodeDecodeError) as e:
+            raise SerializationError(
+                f"Failed to incrementally load YAML: {e}",
+                format_name=self.format_name,
+                original_error=e
+            ) from e
 

exonware/xwsystem/io/serialization/parsers/__init__.py

@@ -0,0 +1,15 @@
+"""JSON Parser abstraction layer for pluggable performance optimizations.
+
+This module provides a pluggable parser system that allows switching between
+different JSON parsing implementations (stdlib, orjson, etc.) for performance.
+"""
+
+from .base import IJsonParser
+from .registry import get_parser, get_best_available_parser, register_parser
+
+__all__ = [
+    'IJsonParser',
+    'get_parser',
+    'get_best_available_parser',
+    'register_parser',
+]

exonware/xwsystem/io/serialization/parsers/base.py

@@ -0,0 +1,59 @@
+"""Base JSON parser interface."""
+
+from abc import ABC, abstractmethod
+from typing import Any, Union
+
+
+class IJsonParser(ABC):
+    """Abstract JSON parser interface for pluggable implementations."""
+    
+    @abstractmethod
+    def loads(self, s: Union[str, bytes]) -> Any:
+        """
+        Parse JSON string/bytes to Python object.
+        
+        Args:
+            s: JSON string or bytes
+            
+        Returns:
+            Parsed Python object
+        """
+        pass
+    
+    @abstractmethod
+    def dumps(self, obj: Any, **kwargs) -> Union[str, bytes]:
+        """
+        Serialize Python object to JSON.
+        
+        Args:
+            obj: Python object to serialize
+            **kwargs: Serialization options (ensure_ascii, indent, etc.)
+            
+        Returns:
+            JSON string or bytes
+        """
+        pass
+    
+    @property
+    @abstractmethod
+    def parser_name(self) -> str:
+        """Parser identifier (e.g., 'standard', 'orjson')."""
+        pass
+    
+    @property
+    @abstractmethod
+    def tier(self) -> int:
+        """
+        Performance tier:
+        0 = stdlib (baseline)
+        1 = orjson (3-4x faster)
+        2 = Rust extension (5-7x faster, future)
+        3 = Pure Rust core (6-8x faster, future)
+        """
+        pass
+    
+    @property
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Check if parser is available (dependencies installed)."""
+        pass

exonware/xwsystem/io/serialization/parsers/hybrid_parser.py

@@ -0,0 +1,61 @@
+"""Hybrid parser: msgspec for reading, orjson for writing (direct, no try/catch)."""
+
+from typing import Any, Union
+import msgspec  # Direct import for reading
+import orjson  # Direct import for writing
+
+from .base import IJsonParser
+
+
+class HybridParser(IJsonParser):
+    """
+    Hybrid parser - fastest combination:
+    - msgspec for reading (1.36x faster than orjson)
+    - orjson for writing (2.27x faster than msgspec)
+    
+    Direct imports (no try/catch) - assumes both are available.
+    """
+    
+    @property
+    def parser_name(self) -> str:
+        return "hybrid"
+    
+    @property
+    def tier(self) -> int:
+        return 1
+    
+    @property
+    def is_available(self) -> bool:
+        return True  # Assumes both msgspec and orjson are available
+    
+    def loads(self, s: Union[str, bytes]) -> Any:
+        """Parse JSON using msgspec.json.decode() - fastest for reading."""
+        if isinstance(s, str):
+            s = s.encode("utf-8")
+        # msgspec.json.decode accepts bytes directly
+        return msgspec.json.decode(s)
+    
+    def dumps(self, obj: Any, **kwargs) -> Union[str, bytes]:
+        """Serialize JSON using orjson.dumps() - fastest for writing."""
+        option = 0
+        
+        # orjson options
+        if not kwargs.get("ensure_ascii", True):
+            # orjson always outputs UTF-8, so ensure_ascii=False is default
+            pass
+        
+        # Handle indent (orjson doesn't support indent directly)
+        indent = kwargs.get("indent", None)
+        if indent:
+            # For pretty printing, use orjson.OPT_INDENT_2
+            option |= orjson.OPT_INDENT_2
+        
+        # Sort keys
+        if kwargs.get("sort_keys", False):
+            option |= orjson.OPT_SORT_KEYS
+        
+        result = orjson.dumps(obj, option=option)
+        
+        # Return as bytes (orjson returns bytes)
+        # Caller can decode if string is needed
+        return result