streaming-json-parser 0.1.0__py3-none-any.whl

streaming_json_parser/iterative_state_machine.py

@@ -0,0 +1,552 @@
+import json
+import re
+from typing import Any, Optional, Union
+
+# State constants for the iterative parser
+S_INIT, S_OBJ_START, S_OBJ_KEY, S_OBJ_COLON, S_OBJ_VALUE, S_OBJ_COMMA, \
+S_ARR_START, S_ARR_VALUE, S_ARR_COMMA, S_COMPLETE, S_ERROR = range(11)
+
+class IterativeStateMachine:
+    def __init__(self):
+        # Pre-compile regex for finding unquoted keys for efficiency
+        # Allows alphanumeric characters, _, -, . as part of the key
+        self.__key_pattern = re.compile(r"([a-zA-Z0-9_.-]+)")
+
+    def parse_iterative_partial(self, s: str) -> tuple[Optional[Union[dict[str, Any], list[Any]]], int]:
+        """
+        An iterative state-machine based parser for potentially incomplete or
+        slightly non-standard JSON (unquoted keys, single quotes).
+        Attempts to parse one top-level object or array.
+
+        Args:
+            s: The string buffer containing JSON data (or partial data).
+
+        Returns:
+            A tuple containing:
+            - The parsed object (dict) or array (list), or None if parsing fails early.
+            - The index in the string 's' up to which parsing consumed input.
+              This index might point to the end of a complete structure, or the
+              point where parsing stopped due to incomplete data or an error.
+        """
+        state_stack = [S_INIT]
+        # Holds the nested dicts/lists currently being built
+        container_stack: list[Union[dict[str, Any], list[Any]]] = []
+        # Stores the key for the current object value being parsed
+        current_key: Optional[str] = None
+        # The root object or list being parsed
+        root: Optional[Union[dict[str, Any], list[Any]]] = None
+        i = 0 # Current parsing index in string 's'
+        len_s = len(s)
+
+        while i < len_s:
+            current_state = state_stack[-1]
+            char = s[i]
+
+            # Skip Whitespace efficiently
+            if char.isspace():
+                i += 1
+                continue
+
+            try:
+                # --- State Machine Logic ---
+
+                # Initial state: Expect start of object '{' or array '['
+                if current_state == S_INIT:
+                    if char == '{':
+                        root = {}
+                        container_stack.append(root)
+                        state_stack[-1] = S_OBJ_START # Transition to object start state
+                        i += 1
+                    elif char == '[':
+                        root = []
+                        container_stack.append(root)
+                        state_stack[-1] = S_ARR_START # Transition to array start state
+                        i += 1
+                    else:
+                        # Error: Input doesn't start with '{' or '['
+                        state_stack.append(S_ERROR)
+                        break # Stop parsing
+
+                # === OBJECT STATES ===
+
+                # Inside object, after '{' or comma: Expect key or '}'
+                elif current_state == S_OBJ_START:
+                    if char == '}':
+                        # Empty object or end of object
+                        if not container_stack:
+                             # Should be impossible if state logic is correct
+                            state_stack.append(S_ERROR)
+                            break
+                        state_stack.pop() # Pop S_OBJ_START state
+                        container_stack.pop() # Pop the completed object
+                        i += 1
+                        if not state_stack:
+                            # If stack is now empty, we finished the root object
+                            state_stack.append(S_COMPLETE)
+                            break
+                        # Otherwise, we were in a nested object, continue parsing parent
+                    else:
+                        # Expecting an object key next
+                        state_stack[-1] = S_OBJ_KEY
+                        # Do not increment 'i' here, S_OBJ_KEY needs to process current char
+
+                # Expecting an object key (quoted or unquoted)
+                elif current_state == S_OBJ_KEY:
+                    key = None
+                    start_i = i
+                    if char == '"':
+                        key, i = self.__parse_partial_string(s, i)
+                    elif char == "'": # Non-standard: single quotes
+                        key, i = self.__parse_single_quoted_string(s, i)
+                    else: # Non-standard: unquoted key
+                        key, i = self.__parse_unquoted_key(s, i)
+
+                    # Check if a key was successfully parsed and index advanced
+                    if key is not None and i > start_i:
+                        current_key = key
+                        state_stack[-1] = S_OBJ_COLON # Transition: Expect colon next
+                    else:
+                        # Failed to parse a valid key at this position
+                        state_stack.append(S_ERROR)
+                        break # Stop parsing
+
+                # Expecting the colon ':' after an object key
+                elif current_state == S_OBJ_COLON:
+                    if char == ':':
+                        state_stack[-1] = S_OBJ_VALUE # Transition: Expect value next
+                        i += 1
+                    else:
+                        # Error: Missing colon after key
+                        state_stack.append(S_ERROR)
+                        break # Stop parsing
+
+                # Expecting a value (primitive, object, or array) for the current key
+                elif current_state == S_OBJ_VALUE:
+                    value_parsed = False
+                    start_i = i
+                    # Get the object this key-value pair belongs to
+                    parent_obj = container_stack[-1]
+
+                    # Basic check for state consistency
+                    if not isinstance(parent_obj, dict) or current_key is None:
+                        state_stack.append(S_ERROR)
+                        break # Should not happen in correct state flow
+
+                    # Check for nested object start
+                    if char == '{':
+                        new_obj = {}
+                        parent_obj[current_key] = new_obj
+                        # Transition parent state to expect comma/close brace
+                        state_stack[-1] = S_OBJ_COMMA
+                        # Push new object and its state onto stacks
+                        container_stack.append(new_obj)
+                        state_stack.append(S_OBJ_START)
+                        i += 1
+                        value_parsed = True
+                    # Check for nested array start
+                    elif char == '[':
+                        new_arr = []
+                        parent_obj[current_key] = new_arr
+                        # Transition parent state to expect comma/close brace
+                        state_stack[-1] = S_OBJ_COMMA
+                        # Push new array and its state onto stacks
+                        container_stack.append(new_arr)
+                        state_stack.append(S_ARR_START)
+                        i += 1
+                        value_parsed = True
+                    else:
+                        # Attempt to parse a primitive value (string, number, literal)
+                         value, next_i = self.__parse_primitive_value(s, i)
+                         if next_i > i: # Check if parsing advanced the index
+                             parent_obj[current_key] = value
+                             state_stack[-1] = S_OBJ_COMMA # Expect comma/close brace next
+                             i = next_i # Update index to after the primitive
+                             value_parsed = True
+                         # else: primitive parsing failed, handled below
+
+                    if not value_parsed:
+                        # Error: Could not parse a valid value after the colon
+                        state_stack.append(S_ERROR)
+                        break # Stop parsing
+
+                # After an object value: Expect comma ',' or closing brace '}'
+                elif current_state == S_OBJ_COMMA:
+                     if char == ',':
+                         state_stack[-1] = S_OBJ_KEY # Transition: Expect another key
+                         i += 1
+                     elif char == '}':
+                         # End of the current object
+                         if not container_stack:
+                             state_stack.append(S_ERROR) # Error: Unbalanced braces
+                             break
+                         state_stack.pop() # Pop S_OBJ_COMMA state
+                         container_stack.pop() # Pop the completed object
+                         i += 1
+                         if not state_stack:
+                             # Finished the root object
+                             state_stack.append(S_COMPLETE)
+                             break
+                         # Else, continue processing parent container
+                     else:
+                         # Error: Expected ',' or '}' after value
+                         state_stack.append(S_ERROR)
+                         break # Stop parsing
+
+                # === ARRAY STATES ===
+
+                # Inside array, after '[' or comma: Expect value or ']'
+                elif current_state == S_ARR_START:
+                    if char == ']':
+                        # Empty array or end of array
+                        if not container_stack:
+                            state_stack.append(S_ERROR) # Error: Unbalanced brackets
+                            break
+                        state_stack.pop() # Pop S_ARR_START state
+                        container_stack.pop() # Pop the completed array
+                        i += 1
+                        if not state_stack:
+                            # Finished the root array
+                            state_stack.append(S_COMPLETE)
+                            break
+                        # Else, continue processing parent container
+                    else:
+                        # Expecting an array value next
+                        state_stack[-1] = S_ARR_VALUE
+                        # Do not increment 'i', S_ARR_VALUE needs to process current char
+
+                # Expecting an array value (primitive, object, or array)
+                elif current_state == S_ARR_VALUE:
+                    value_parsed = False
+                    start_i = i
+                    # Get the array this value belongs to
+                    parent_arr = container_stack[-1]
+
+                    # Basic check for state consistency
+                    if not isinstance(parent_arr, list):
+                         state_stack.append(S_ERROR)
+                         break # Should not happen
+
+                    # Check for nested object start
+                    if char == '{':
+                        new_obj = {}
+                        parent_arr.append(new_obj)
+                        # Transition parent state to expect comma/close bracket
+                        state_stack[-1] = S_ARR_COMMA
+                        # Push new object and its state
+                        container_stack.append(new_obj)
+                        state_stack.append(S_OBJ_START)
+                        i += 1
+                        value_parsed = True
+                    # Check for nested array start
+                    elif char == '[':
+                        new_arr = []
+                        parent_arr.append(new_arr)
+                        # Transition parent state to expect comma/close bracket
+                        state_stack[-1] = S_ARR_COMMA
+                        # Push new array and its state
+                        container_stack.append(new_arr)
+                        state_stack.append(S_ARR_START)
+                        i += 1
+                        value_parsed = True
+                    else:
+                        # Attempt to parse a primitive value
+                        value, next_i = self.__parse_primitive_value(s, i)
+                        if next_i > i: # Check if parsing advanced the index
+                            parent_arr.append(value)
+                            state_stack[-1] = S_ARR_COMMA # Expect comma/close bracket next
+                            i = next_i # Update index
+                            value_parsed = True
+                        # else: primitive parsing failed, handled below
+
+                    if not value_parsed:
+                        # Error: Could not parse a valid value in array
+                        state_stack.append(S_ERROR)
+                        break # Stop parsing
+
+                # After an array value: Expect comma ',' or closing bracket ']'
+                elif current_state == S_ARR_COMMA:
+                     if char == ',':
+                         state_stack[-1] = S_ARR_VALUE # Transition: Expect another value
+                         i += 1
+                     elif char == ']':
+                         # End of the current array
+                         if not container_stack:
+                             state_stack.append(S_ERROR) # Error: Unbalanced brackets
+                             break
+                         state_stack.pop() # Pop S_ARR_COMMA state
+                         container_stack.pop() # Pop the completed array
+                         i += 1
+                         if not state_stack:
+                             # Finished the root array
+                             state_stack.append(S_COMPLETE)
+                             break
+                         # Else, continue processing parent container
+                     else:
+                         # Error: Expected ',' or ']' after array element
+                         state_stack.append(S_ERROR)
+                         break # Stop parsing
+
+                # === END/ERROR STATES ===
+
+                # Parsing completed successfully for the root element
+                elif current_state == S_COMPLETE:
+                    break # Exit main loop
+
+                # An error occurred during parsing
+                elif current_state == S_ERROR:
+                    break # Exit main loop
+
+                # Catchall for unknown states (should not happen)
+                else:
+                     print(f"Warning: Encountered unknown parser state {current_state}")
+                     state_stack.append(S_ERROR)
+                     break # Stop parsing
+
+            except IndexError:
+                 # Catch errors accessing state_stack or container_stack if they become empty unexpectedly
+                 print(f"Error: Stack underflow at index {i}, state {current_state}. Likely malformed JSON.")
+                 state_stack.append(S_ERROR)
+                 break
+            except Exception as inner_ex:
+                 # Catch any other unexpected errors during state processing
+                 print(f"Error during iterative parse step: {inner_ex} at index {i}, state {current_state}")
+                 state_stack.append(S_ERROR)
+                 break # Stop parsing on unexpected exceptions
+
+        # Determine the final state and return result
+        final_state = state_stack[-1]
+
+        if final_state == S_COMPLETE:
+            # Parsed a complete object/array successfully
+            return root, i
+        elif final_state == S_ERROR:
+             # Parsing stopped due to an error. Return whatever was parsed
+             # up to the error point (might be None or partial structure)
+             # 'i' will be the index where the error occurred.
+            return root, i
+        else:
+            # Loop finished because end of input string 's' was reached,
+            # but the JSON structure wasn't 'complete' (e.g., missing closing brace).
+            # Return the partially parsed structure and the final index 'i'.
+            return root, i
+    
+    def __parse_primitive_value(self, s: str, i: int) -> tuple[Any, int]:
+         """
+         Attempts to parse any primitive JSON value (string, number, literal)
+         starting at s[i]. Also handles non-standard single quotes.
+         Returns the parsed value and the index after it, or (None, i) on failure.
+         """
+         if i >= len(s):
+             # Cannot parse if index is out of bounds
+             return None, i
+
+         char = s[i]
+
+         # Check for string start (double quote)
+         if char == '"':
+             return self.__parse_partial_string(s, i)
+
+         # Check for non-standard string start (single quote)
+         if char == "'":
+             return self.__parse_single_quoted_string(s, i)
+
+         # Check for number start (digit or minus sign)
+         if char in "-0123456789":
+             return self.__parse_partial_number(s, i)
+
+         # Check for literals (true, false, null)
+         val, next_i = self.__parse_partial_literal(s, i)
+         if next_i > i:
+             # A literal was successfully parsed
+             return val, next_i
+
+         # If none of the above matched, it's not a recognizable primitive start
+         return None, i # Indicate failure by returning original index
+
+    def __parse_unquoted_key(self, s: str, i: int) -> tuple[Optional[str], int]:
+         """
+         Attempts to parse an unquoted object key starting at s[i]. Non-standard.
+         Uses the pre-compiled regex `_key_pattern`.
+         Returns the key string and the index after it, or (None, i) if no match.
+         """
+         match = self.__key_pattern.match(s, i)
+         if match:
+             # Key must not be followed immediately by ":" without whitespace,
+             # handle ':' separation in the main loop.
+             key = match.group(1)
+             return key, match.end()
+         else:
+             # No match for the unquoted key pattern
+             return None, i
+    
+    def __parse_single_quoted_string(self, s: str, i: int) -> tuple[str, int]:
+        """
+        Parses a single-quoted string starting at s[i]. Non-standard JSON.
+        Assumes s[i] == "'". Handles limited escapes (\' and \\).
+        Returns the parsed string content and the index after the closing quote,
+        or the end of the string if unterminated.
+        """
+        i += 1 # skip opening '
+        result = []
+        escape = False
+
+        while i < len(s):
+            ch = s[i]
+
+            if escape:
+                if ch == "'":
+                    result.append("'")
+                elif ch == '\\':
+                    result.append('\\')
+                else:
+                    # Pass through other characters following a backslash
+                    result.append('\\')
+                    result.append(ch)
+                escape = False
+            elif ch == "\\":
+                escape = True
+            elif ch == "'":
+                # End of string found
+                return "".join(result), i + 1
+            else:
+                result.append(ch)
+
+            i += 1
+
+        # Unterminated single-quoted string
+        return "".join(result), i
+
+    def __parse_partial_number(self, s: str, i: int) -> tuple[Any, int]:
+        """
+        Parses a number (int or float) starting at s[i].
+        Uses regex for robust matching of JSON number format.
+        Returns the parsed number (int or float) and the index after the number,
+        or (None, i) if no valid number start is found.
+        """
+        # Regex for standard JSON numbers (integer and floating point)
+        # Allows leading minus, optional fractional part, optional exponent
+        num_match = re.match(r"-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?", s[i:])
+
+        if num_match:
+            num_str = num_match.group(0)
+            end_i = i + len(num_str)
+            try:
+                # Use json.loads on the matched string for reliable conversion
+                # This correctly handles int vs float detection.
+                return json.loads(num_str), end_i
+            except json.JSONDecodeError:
+                # This should theoretically not happen if the regex is correct,
+                # but acts as a safeguard. Return the raw string maybe?
+                # Or indicate failure. Let's return failure indicator.
+                return None, i # Indicate parsing failed despite regex match
+        else:
+            # No valid number pattern matched at the current position
+            return None, i
+
+    def __parse_partial_literal(self, s: str, i: int) -> tuple[Any, int]:
+        """
+        Parses JSON literals (true, false, null) starting at s[i].
+        Checks for word boundaries to avoid partial matches (e.g., 'trueish').
+        Returns the literal value (True, False, None) and the index after it,
+        or (None, i) if no literal is matched.
+        """
+        len_s = len(s)
+
+        # Check for 'true'
+        if s.startswith("true", i):
+            end_i = i + 4
+            # Check if it's the end of the string or followed by a non-alphanumeric char
+            if end_i == len_s or not s[end_i].isalnum():
+                return True, end_i
+
+        # Check for 'false'
+        if s.startswith("false", i):
+            end_i = i + 5
+            # Check for word boundary
+            if end_i == len_s or not s[end_i].isalnum():
+                return False, end_i
+
+        # Check for 'null'
+        if s.startswith("null", i):
+            end_i = i + 4
+            # Check for word boundary
+            if end_i == len_s or not s[end_i].isalnum():
+                return None, end_i
+
+        # No literal matched at this position
+        return None, i
+
+    def __parse_partial_string(self, s: str, i: int) -> tuple[str, int]:
+        """
+        Parses a double-quoted string starting at s[i].
+        Assumes s[i] == '"'. Handles standard JSON escapes.
+        Returns the parsed string content and the index after the closing quote,
+        or the end of the string if unterminated.
+        """
+        i += 1 # skip opening "
+        result = []
+        escape = False
+
+        while i < len(s):
+            ch = s[i]
+
+            # Handle escape sequences
+            if escape:
+                if ch == 'b':
+                    result.append('\b')
+                elif ch == 'f':
+                    result.append('\f')
+                elif ch == 'n':
+                    result.append('\n')
+                elif ch == 'r':
+                    result.append('\r')
+                elif ch == 't':
+                    result.append('\t')
+                elif ch == '"':
+                    result.append('"')
+                elif ch == '\\':
+                    result.append('\\')
+                elif ch == '/':
+                    result.append('/') # Allowed escape, often seen
+                elif ch == 'u':
+                    # Unicode escape (basic handling)
+                    if i + 4 < len(s):
+                        try:
+                            hex_code = s[i + 1 : i + 5]
+                            result.append(chr(int(hex_code, 16)))
+                            i += 4 # Skip hex digits
+                        except ValueError:
+                            # Invalid hex code, keep the original sequence
+                            result.append('\\u')
+                            result.append(s[i + 1 : i + 5])
+                            i += 4
+                    else:
+                        # Incomplete escape sequence at end of buffer
+                        result.append('\\u')
+                        # Append remaining characters if any
+                        if i + 1 < len(s):
+                            result.append(s[i + 1 :])
+                        i = len(s) # Move index to end
+                        break # Exit loop as escape is incomplete
+                else:
+                    # Pass through unknown escape sequences
+                    result.append('\\')
+                    result.append(ch)
+                escape = False
+            elif ch == "\\":
+                escape = True
+            elif ch == '"':
+                # End of string found
+                return "".join(result), i + 1
+            else:
+                # Handle control characters (U+0000 to U+001F)
+                # Escape them to \uXXXX format if not already escaped
+                if 0x00 <= ord(ch) <= 0x1F:
+                    result.append(f'\\u{ord(ch):04x}')
+                result.append(ch)
+
+            i += 1
+
+        # If loop finishes without finding closing quote, string is unterminated.
+        # Return the partial result found so far.
+        return "".join(result), i

streaming_json_parser/streaming_json_parser.py

@@ -0,0 +1,173 @@
+import json
+import re
+from typing import Any
+
+from src.streaming_json_parser.iterative_state_machine import \
+    IterativeStateMachine
+
+
+class StreamingJsonParser:
+    """
+    A parser designed to handle potentially incomplete or slightly malformed
+    JSON streams, attempting to extract valid JSON objects.
+    """
+    def __init__(self):
+        self.__buffer: str = ""
+    
+    def consume(self, data: str) -> None:
+        """
+        Adds new data chunks to the internal buffer after escaping invalid chars.
+        """
+        # Escape control characters before adding to buffer
+        # This helps prevent issues if these chars appear outside strings
+        if not self.__is_string(data):
+            return
+
+        escaped_data = self.__escape_invalid_control_chars(data)
+        if escaped_data:
+            self.__buffer += escaped_data
+
+    def get(self) -> dict[str, Any]:
+        """
+        Attempts to parse and return the first complete JSON object found
+        in the buffer. Removes the parsed object (and preceding non-JSON data)
+        from the buffer. Returns an empty dict if no complete object is found.
+        """
+        # --- Initial buffer cleanup ---
+        # Remove potential BOM (Byte Order Mark) and leading whitespace
+        self.__buffer = self.__buffer.lstrip("\ufeff").lstrip()
+
+        # Find the first opening brace '{' which signifies a potential JSON object start
+        start_index = self.__buffer.find("{")
+        if start_index == -1:
+            # No object start found, clear buffer if it contains only whitespace/junk
+            if not self.__buffer.strip():
+                 self.__buffer = ""
+            # Otherwise, keep buffer as it might be incomplete start of something else
+            return {}
+        elif start_index > 0:
+            # Discard anything before the first '{'
+            self.__buffer = self.__buffer[start_index:]
+
+        # If buffer became empty after stripping/finding '{', return empty
+        if not self.__buffer:
+            return {}
+
+        # --- Attempt 1: Use standard json.raw_decode for well-formed JSON ---
+        # This is efficient for standard JSON.
+        try:
+            decoder = json.JSONDecoder()
+            # raw_decode parses one JSON value and returns it and the index where it stopped
+            obj, idx = decoder.raw_decode(self.__buffer)
+
+            # Check if the decoded item is a dictionary (object)
+            if isinstance(obj, dict):
+                # Successfully parsed a standard JSON object
+                self.__buffer = self.__buffer[idx:] # Remove parsed part from buffer
+                self.__clean_buffer_after_parse() # Clean up potential leading junk
+                return obj
+            else:
+                # Parsed something, but it wasn't an object (e.g., list, primitive)
+                # Discard the parsed part and try again (or let partial parser handle it)
+                self.__buffer = self.__buffer[idx:]
+                # Fall through to the partial parser attempt
+        except json.JSONDecodeError:
+            # raw_decode failed, likely due to incomplete or malformed JSON.
+            # Proceed to the more lenient iterative partial parser.
+            pass
+        except Exception:
+            # Catch other potential errors during raw_decode
+            # print(f"Unexpected error during raw_decode: {e}") # Optional logging
+            pass # Fall through to partial parser
+
+        # --- Attempt 2: Use the Iterative Partial Parser ---
+        # This handles incomplete data, unquoted keys, single quotes etc.
+        self.__buffer = self.__buffer.lstrip() # Ensure no leading whitespace
+
+        # Re-check if buffer starts with '{' after potential modification from attempt 1
+        if not self.__buffer.startswith('{'):
+            start_index = self.__buffer.find("{")
+            if start_index == -1:
+                 # No '{' found at all anymore
+                self.__buffer = ""
+                return {}
+            else:
+                 # Discard leading content if any before '{'
+                self.__buffer = self.__buffer[start_index:]
+
+        # Re-check if buffer is empty
+        if not self.__buffer:
+            return {}
+
+        # Run the iterative state-machine parser
+        iterative_state_machine = IterativeStateMachine()
+        parsed_obj, consumed_idx = iterative_state_machine.parse_iterative_partial(self.__buffer)
+
+        # Update buffer based on how much was consumed
+        if 0 <= consumed_idx <= len(self.__buffer):
+            # Successfully parsed or partially parsed, remove consumed part
+            self.__buffer = self.__buffer[consumed_idx:]
+        else:
+            # Indicates an error or unexpected state, clear buffer to be safe
+            self.__buffer = ""
+
+        # Clean buffer again after partial parse attempt
+        self.__clean_buffer_after_parse()
+
+        # Return the parsed object only if it's a dictionary
+        if isinstance(parsed_obj, dict):
+            return parsed_obj
+        else:
+            # The partial parser might return lists or None in some edge cases/errors
+            return {}
+
+    def __escape_invalid_control_chars(self, s: str) -> str:
+        """
+        Escapes control characters (U+0000 to U+001F) that are invalid in JSON
+        unless escaped. Standard escapes like \n, \t are preserved.
+        """
+        # Mapping for standard JSON escapes within the control character range
+        escape_map = {
+            "\b": "\\b",  # Backspace (U+0008)
+            "\f": "\\f",  # Form feed (U+000C)
+            "\n": "\\n",  # Line feed (U+000A)
+            "\r": "\\r",  # Carriage return (U+000D)
+            "\t": "\\t"   # Horizontal tab (U+0009)
+        }
+
+        # Replacement function for re.sub
+        def replace(match: re.Match[str]) -> str:
+            ch = match.group(0)
+            # Use standard escape if available, otherwise use \uXXXX format
+            return escape_map.get(ch, f"\\u{ord(ch):04x}")
+
+        # Regex to find all characters from U+0000 to U+001F
+        return re.sub(r"[\x00-\x1F]", replace, s)
+
+    def __clean_buffer_after_parse(self):
+        """
+        Helper to clean the buffer after a successful parse.
+        Removes leading whitespace and searches for the next potential object start.
+        If no '{' is found, clears the buffer if it only contains whitespace.
+        """
+        self.__buffer = self.__buffer.lstrip()
+        next_obj_start = self.__buffer.find("{")
+
+        if next_obj_start > 0:
+            # Found another potential object start, discard text before it
+            self.__buffer = self.__buffer[next_obj_start:]
+        elif next_obj_start == -1:
+            # No more '{' found. If buffer is just whitespace, clear it.
+            if not self.__buffer.strip():
+                self.__buffer = ""
+            # Otherwise, keep the buffer content (might be start of next partial object)
+
+    def __is_string(self, data: Any) -> bool:
+        """
+        Helper to check if the provided data is a string.
+        Args:
+            data: The data to check.
+        Returns:
+            True if data is a string, False otherwise.
+        """
+        return isinstance(data, str)

streaming_json_parser-0.1.0.dist-info/METADATA

@@ -0,0 +1,156 @@
+Metadata-Version: 2.4
+Name: streaming-json-parser
+Version: 0.1.0
+Summary: A streaming JSON parser that processes JSON data incrementally, handling partial states. Useful for incrementally parsing partial responses from streaming outputs of Large Language Models (LLMs).
+Author-email: Aramis Facchinetti <aramis.facchinetti16@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/aramisfacchinetti/streaming-json-parser
+Project-URL: Repository, https://github.com/aramisfacchinetti/streaming-json-parser
+Keywords: streaming,json,parser,llm,large language model,incremental parsing
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: test
+Requires-Dist: pytest>=8.3.5; extra == "test"
+Dynamic: license-file
+
+# Streaming JSON Parser
+
+## Objective
+
+This Python module implements a streaming JSON parser designed to process JSON data incrementally. The primary goal is to handle potentially incomplete JSON data streams, such as those produced by Large Language Models (LLMs), and return the current state of the parsed object at any time.
+
+## Requirements Subset
+
+The parser is specifically designed for a subset of JSON where:
+
+- Values consist solely of **strings** and **objects**.
+- **Escape sequences** in strings are not expected (though the implementation handles them).
+- **Duplicate keys** in objects are not expected (though the implementation may tolerate them, typically keeping the last value).
+
+## Features
+
+- **Incremental Parsing:** Consumes JSON data in chunks via the `consume()` method.
+- **Partial State Retrieval:** The `get()` method returns the currently parsed JSON object state, even if the input stream is incomplete.
+- **Partial String Values:** Returns partial string values as they are received (e.g., `{"key": "val` is valid partial state).
+- **Key Handling:** Keys are only included in the returned object once their value type (string or object start) is identified.
+- **Robustness:** Attempts to parse standard JSON efficiently and falls back to a more lenient state-machine parser for incomplete or slightly non-standard input.
+- **Non-Standard JSON:** Tolerates some non-standard features like unquoted keys and single-quoted strings.
+- **Error Handling:** Attempts to recover from invalid characters or find the first valid JSON object within the buffer.
+- **Support for Primitives & Arrays:** Although the requirements focused on strings and objects, the implementation also handles numbers, booleans, null, and arrays as values within objects.
+
+## Implementation Approach
+
+1.  **Buffering:** The `consume()` method appends incoming data chunks to an internal string buffer after escaping potentially invalid control characters.
+2.  **Parsing (`get()`):**
+    - The buffer is first cleaned by removing leading whitespace and any characters before the first `{`.
+    - It attempts parsing using `json.raw_decode` for speed and standard compliance. If a dictionary is successfully decoded, it's returned, and the consumed portion is removed from the buffer.
+    - If `raw_decode` fails (due to incomplete data, syntax errors, or non-standard features), it falls back to the `IterativeStateMachine`.
+    - The `IterativeStateMachine` parses the buffer character by character, maintaining state to handle nested structures, different value types (including non-standard ones like unquoted keys), and partial inputs.
+    - The `get()` method returns the dictionary parsed by either method and updates the buffer, removing the parsed object and any leading garbage before the _next_ potential object. If no complete object can be parsed, an empty dictionary is returned.
+
+## Assumptions and Extensions
+
+The implementation makes the following assumptions or extends the requirements:
+
+1.  **Handling of Additional Primitive Types:** Supports numbers (int, float), booleans (`true`, `false`), and `null` as values, beyond the specified strings and objects.
+2.  **Handling of Arrays:** Supports JSON arrays (`[...]`) as values within objects and can parse them, although `get()` only returns top-level _objects_ (`dict`).
+3.  **Non-Standard JSON Support:** Tolerates and parses:
+    - Unquoted object keys (e.g., `{key: "value"}`).
+    - Single-quoted strings (e.g., `{'key': 'value'}`).
+4.  **Escape Sequence Handling:** Actively handles standard JSON escape sequences (e.g., `\n`, `\"`) and Unicode escapes (`\uXXXX`) within strings, although they were "not expected".
+5.  **Control Character Handling:** Escapes invalid JSON control characters (U+0000 to U+001F) found _outside_ of strings in the input buffer using `\uXXXX` format during `consume`.
+6.  **Error Recovery/Robustness:** Discards leading non-JSON data before the first `{` and attempts to parse the first valid object found. Handles multiple objects in the buffer sequentially across `get()` calls.
+7.  **Duplicate Keys:** Does not explicitly prevent duplicate keys; standard Python dictionary behavior (last key wins) likely applies.
+8.  **Efficiency Strategy:** Uses `json.raw_decode` first, falling back to a custom parser only when necessary.
+9.  **Input Type:** `consume` expects string input; other types are ignored.
+
+## Algorithmic Complexity
+
+The efficiency of the `StreamingJsonParser` depends on the method being called and the nature of the input data stream.
+
+- **`consume(buffer: str)`:**
+
+  - **Time Complexity:** Primarily involves appending the new `buffer` (length `k`) to the internal buffer and performing basic character escaping. This is typically **O(k)**. String concatenation in Python can sometimes be O(N+k) where N is the current buffer size, but often optimized closer to O(k) amortized.
+  - **Space Complexity:** Increases the internal buffer size by O(k).
+
+- **`get()`:**
+
+  - **Time Complexity:**
+    - **Fast Path (`json.raw_decode`):** If the buffer starts with a complete, standard JSON object of size `P`, Python's built-in decoder is used. This is generally efficient, expected to be around **O(P)**.
+    - **Fallback Path (`IterativeStateMachine`):** If `raw_decode` fails (due to incomplete data or non-standard syntax), the custom state machine parses the buffer character by character. In the worst case, it might need to scan a significant portion of the buffer (size `B'`). The complexity is dominated by this scan and subsequent buffer slicing, making it roughly **O(B')**.
+    - **Overall:** The complexity varies. It's close to O(P) when complete objects are readily available and standard, and approaches O(B') when parsing incomplete or non-standard streams requires the iterative fallback.
+  - **Space Complexity:** Does not inherently allocate significant additional space beyond the internal representation of the parsed object being returned. The main space usage comes from the internal buffer managed by `consume`.
+
+- **Overall Space Complexity:** The primary factor is the internal buffer. In the worst case (e.g., a very large stream is consumed without any complete objects being parsed and removed by `get()`), the space complexity can be **O(T)**, where T is the total size of the streamed data received so far. In typical usage where `get()` successfully parses and removes objects, the buffer size stays manageable.
+
+## Usage
+
+```python
+# Import the class
+from streaming_json_parser import StreamingJsonParser
+
+# Initialize the parser
+parser = StreamingJsonParser()
+
+# Consume JSON data chunks
+parser.consume('{"name": "Example", "data": {"val') # Partial object value
+parser.consume('ue": "stream"}')                  # Complete the object
+
+# Get the current state of the parsed object
+# This will return the first complete object found.
+current_object = parser.get()
+print(current_object)
+# Output: {'name': 'Example', 'data': {'value': 'stream'}}
+
+# The buffer is cleared/updated after get(), ready for the next object
+parser.consume('{"next": "object"}')
+next_object = parser.get()
+print(next_object)
+# Output: {'next': 'object'}
+
+# Example with partial string value
+parser = StreamingJsonParser()
+parser.consume('{"key": "partial string')
+partial_state = parser.get()
+print(partial_state)
+# Output: {'key': 'partial string'}
+
+parser.consume(' complete"}')
+complete_state = parser.get()
+print(complete_state)
+# Output: {'key': 'partial string complete'}
+```
+
+## Setup
+
+To use this parser and run the tests, you need to install the dependencies:
+
+```bash
+pip install -r requirements.txt
+```
+
+The `requirements.txt` file includes:
+
+- `pytest`
+- `pytest-cov`
+
+## Testing
+
+Unit tests are provided in `test_streaming_json_parser.py`. You can run them using `pytest`:
+
+```bash
+pytest
+```

streaming_json_parser-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+streaming_json_parser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+streaming_json_parser/iterative_state_machine.py,sha256=LHM2tMSYwQ26snwXWiLItj5zp0ds9c5gxY2_TVt5EMQ,24555
+streaming_json_parser/streaming_json_parser.py,sha256=N2QJUeRU6V2mQklL8hwS5rOuE9TT3QwzwNxR1SpDY4M,7220
+streaming_json_parser-0.1.0.dist-info/licenses/LICENSE,sha256=LCzjgEJx8tM9dXg1qVkOuj9AdZgqNYdh2q0GGWqW9s0,1075
+streaming_json_parser-0.1.0.dist-info/METADATA,sha256=Kc7ePMzoMmMI3kb8XCkG1vd39COE1ZbuSoVZPa6g0Ic,9208
+streaming_json_parser-0.1.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+streaming_json_parser-0.1.0.dist-info/top_level.txt,sha256=esGYTSNfbg07ZazVHscWNod09hjodlQ8TNYP4sAKYh4,22
+streaming_json_parser-0.1.0.dist-info/RECORD,,

streaming_json_parser-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (78.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

streaming_json_parser-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Aramis Facchinetti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

streaming_json_parser-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+streaming_json_parser