streamjson 1.0.3__py3-none-any.whl

streamjson/__init__.py

@@ -0,0 +1,2 @@
+from .streamjsonwriter import StreamJSONWriter
+from .streamjsonreader import StreamJSONReader

streamjson/streamjsonreader.py

@@ -0,0 +1,126 @@
+import json
+from json import JSONDecodeError
+from typing import Any
+
+
+class StreamJSONReader:
+    def __init__(self, file: str, encoding: str = "utf8", **kwds: Any):
+        """
+        Read objects from a JSON file using a stream. Does not require loading the whole JSON file in memory.
+
+        :param file: The file
+        :param encoding: Determines the encoding
+        :param kwds: Arguments for json.loads()
+        """
+
+        self.__file = file
+        self.__encoding = encoding
+        self.__kwds = kwds
+
+        self.__reader = self.Reader(
+            file=self.__file, encoding=self.__encoding, **self.__kwds
+        )
+
+    def find(self):
+        return self.__reader.find()
+
+    def close(self):
+        self.__reader.close()
+
+    def __enter__(self):
+        return self.__reader
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.__reader.close()
+
+    class Reader:
+        def __init__(self, file: str, encoding: str = "utf8", **kwds: Any):
+            """
+            :param file: The file
+            :param encoding: Determines the encoding
+            :param kwds: Arguments for json.loads()
+            """
+
+            self.__file = file
+            self.__encoding = encoding
+            self.__kwds = kwds
+
+            self.__opened_file = None
+            self.__tmp_lines = ""
+            self.__object_is_pending = False
+            self.__opening_bracket_index = 0
+            self.__size_bytes = 0
+
+        def find(self):
+            """Find JSON object"""
+
+            # Open the file
+            self.__opened_file = open(self.__file, "r", encoding=self.__encoding)
+
+            # Loop through each line in the JSON file and while doing so, yield JSON objects as they are found
+            for line in self.__opened_file:
+                # Append new line to tmp_lines
+                new_line = "".join(line.splitlines())
+                self.__tmp_lines += new_line
+
+                # FIND OPENING BRACKET {
+
+                # Not object_is_pending means there is no opening bracket expecting a closing bracket at the moment
+                # Therefore it is necessary to look for the opening bracket
+                if not self.__object_is_pending:
+                    # Find the opening bracket and its index
+                    self.__opening_bracket_index = self.__tmp_lines.find("{", 0)
+                    # If nothing was found: Clear tmp_lines and continue to try again with the next line from the file
+                    if self.__opening_bracket_index == -1:
+                        self.__tmp_lines = ""
+                        continue
+                    else:
+                        # If the opening bracket was found:
+                        # Set object_is_pending to true meaning a closing bracket is expected next to yield an object
+                        self.__object_is_pending = True
+
+                # FIND CLOSING BRACKET }
+
+                # Get all indices of closing brackets in the string
+                closing_brackets_indices = [
+                    i for i, char in enumerate(self.__tmp_lines) if char == "}"
+                ]
+
+                # Try all closing brackets for possible JSON object completion
+                for closing_bracket_index in closing_brackets_indices:
+                    str_possible_json_object = self.__tmp_lines[
+                        self.__opening_bracket_index : closing_bracket_index + 1
+                    ]
+                    try:
+                        # Try to load the string as a JSON object
+                        json_obj = json.loads(s=str_possible_json_object, **self.__kwds)
+
+                        # Clear part of tmp_lines that was used in the object
+                        self.__tmp_lines = self.__tmp_lines[closing_bracket_index + 1 :]
+
+                        # Set False to start looking for opening bracket next
+                        self.__object_is_pending = False
+
+                        # Yield the JSON object
+                        yield json_obj
+
+                        # Append the size of the object
+                        self.__size_bytes += len(
+                            str_possible_json_object.encode("utf-8")
+                        )
+
+                        break
+                    except JSONDecodeError:
+                        continue
+
+        def get_size(self):
+            """Get size of total strings successfully converted to JSON objects from file in bytes"""
+
+            return self.__size_bytes
+
+        def close(self):
+            """Close the reader"""
+
+            # Close the file
+            if self.__opened_file:
+                self.__opened_file.close()

streamjson/streamjsonwriter.py

@@ -0,0 +1,120 @@
+import json
+import os
+
+
+class StreamJSONWriter:
+    def __init__(self, file: str, indent: int = 2, ensure_ascii: bool = True):
+        """
+        Write objects or arrays to a JSON file using a stream. Useful for when you don't want to read large amounts of
+        data in memory, for example when you need to save large amounts of data from a database to a single JSON file.
+
+        :param file: The file
+        :param indent: Spaces to use at the beginning of line
+        :param ensure_ascii: ascii-only json output (replace non-ascii to \\uNNNN), True by default
+        """
+
+        self.__file = file
+        self.__indent = indent if indent >= 0 else 0
+        self.__ensure_ascii = ensure_ascii
+
+        self.__writer = self.Writer(self.__file, self.__indent, self.__ensure_ascii)
+
+    def send(self, value):
+        self.__writer.send(value)
+
+    def close(self):
+        self.__writer.close()
+
+    def __enter__(self):
+        return self.__writer
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.__writer.close()
+
+    class Writer:
+        def __init__(self, file: str, indent: int, ensure_ascii: bool):
+            """
+            :param file: The file
+            :param indent: Spaces to use at the beginning of line
+            :param ensure_ascii: ascii-only json output (replace non-ascii to \\uNNNN), True by default
+            """
+
+            self.__file = file
+            self.__indent = indent
+            self.__stream_started = False
+            self.__opened_file = None
+            self.__ensure_ascii = ensure_ascii
+
+        def send(self, value):
+            """
+            Send value to file
+
+            :param value: The value to send to the file
+            """
+
+            if not self.__stream_started:
+                self.__stream_started = True
+
+                # Remove the file before writing to it
+                if os.path.exists(self.__file):
+                    os.remove(self.__file)
+
+                # Open the file
+                self.__opened_file = open(self.__file, "a")
+
+                # Add opening bracket at first write
+                self.__opened_file.write("[")
+
+            # Create JSON string from value
+            json_value = json.dumps(
+                json.loads(json.dumps(value, ensure_ascii=self.__ensure_ascii)),
+                indent=self.__indent,
+                ensure_ascii=self.__ensure_ascii,
+            )
+
+            # Indent the whole value
+            json_value_indented = self.__indent_string(json_value)
+
+            self.__opened_file.write(f"\n{json_value_indented},")
+
+        def __indent_string(self, string) -> str:
+            """
+            Indent a string
+
+            :param string: String to indent
+            :return: The indented string
+            """
+
+            def get_indent() -> str:
+                """String representing the indentation"""
+
+                indent_string = ""
+                for x in range(self.__indent):
+                    indent_string += " "
+                return indent_string
+
+            indent = get_indent()
+            return indent + string.replace("\n", "\n" + indent)
+
+        def __remove_last_comma(self, file):
+            """
+            Remove the last comma from the file
+
+            :param file: The file
+            """
+
+            file.seek(file.tell() - 1, os.SEEK_SET)
+            file.truncate()
+
+        def close(self):
+            """Close the writer"""
+
+            if self.__stream_started and self.__opened_file:
+                # Remove last comma
+                self.__remove_last_comma(self.__opened_file)
+
+                # Add the closing bracket
+                self.__opened_file.write("\n]")
+
+                # Close the file
+                self.__opened_file.close()

streamjson-1.0.3.dist-info/METADATA

@@ -0,0 +1,85 @@
+Metadata-Version: 2.4
+Name: streamjson
+Version: 1.0.3
+Summary: Send objects or arrays to a JSON file using a stream. Read objects from a JSON file using a stream.
+Author-email: Zairon Jacobs <zaironjacobs@gmail.com>
+License-Expression: MIT
+Project-URL: homepage, https://github.com/zaironjacobs/streamjson
+Project-URL: download, https://github.com/zaironjacobs/streamjson/archive/v1.0.3.tar.gz
+Keywords: json,stream,write,file,read,objects,arrays
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Natural Language :: English
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# StreamJSON
+
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/streamjson?color=blue)](https://pypi.python.org/pypi/streamjson)
+[![PyPI](https://img.shields.io/pypi/v/streamjson?color=blue)](https://pypi.python.org/pypi/streamjson)
+[![PyPI - License](https://img.shields.io/pypi/l/streamjson)](https://pypi.python.org/pypi/streamjson)
+
+[![tests](https://github.com/zaironjacobs/streamjson/actions/workflows/test.yml/badge.svg)](https://github.com/zaironjacobs/streamjson/actions/workflows/test.yml)
+
+Write objects or arrays to a JSON file using a stream. Useful for when you don't want to read large amounts of data in
+memory, for example when you need to save large amounts of data from a database to a single JSON file.
+
+Read objects from a JSON file using a stream. Does not require loading the whole JSON file in memory.
+
+## Install
+
+```console
+pip install streamjson
+```
+
+## How to write to JSON file
+
+Fetch data from a database or anywhere else and send to a JSON file. The send function takes in a dictionary or a list.
+A new file with the given name will be created, the root of the JSON file is an array by default.
+
+```Python
+from streamjson import StreamJSONWriter
+
+persons = [{'id': '0001', 'first_name': 'John', 'last_name': 'Doe'},
+           {'id': '0002', 'first_name': 'Jane', 'last_name': 'Doe'}]
+
+with StreamJSONWriter('persons.json', indent=2) as writer:
+    for person in persons:
+        writer.send(person)
+```
+
+persons.json:
+
+```JSON
+[
+  {
+    "id": "0001",
+    "first_name": "John",
+    "last_name": "Doe"
+  },
+  {
+    "id": "0002",
+    "first_name": "Jane",
+    "last_name": "Doe"
+  }
+]
+```
+
+## How to read from JSON file
+
+The reader will stream each object from the JSON file.
+
+```Python
+from streamjson import StreamJSONReader
+
+with StreamJSONReader('persons.json') as reader:
+    for obj in reader.find():
+        print(obj)
+```

streamjson-1.0.3.dist-info/RECORD

@@ -0,0 +1,10 @@
+streamjson/__init__.py,sha256=cIqhs_MmO8BF3C-0tEyayiN-J7GW67wA_GyzFNPHZ6I,94
+streamjson/streamjsonreader.py,sha256=_G-TnSgSzqxOrCp-vcYd-9LglxIrsjqtRP1jt9lgoSY,4656
+streamjson/streamjsonwriter.py,sha256=yT6PmCcJANxyYoO7feueHg5Zn33XOKSjGPQhhxY4Dgg,3764
+streamjson-1.0.3.dist-info/licenses/LICENSE,sha256=onT5tOHU-R33_0QLoCcrxyMIc9pPUCEQMYKrc5jSx-A,1069
+tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/test_app.py,sha256=i02yFR0EwnTeqDqYK6l2RjU3zU2_im8qAJk3RRiXGy4,1742
+streamjson-1.0.3.dist-info/METADATA,sha256=vfIElroe27BCThK6KuFbYUHhO7_5UaceWbMWdE6LUkU,2821
+streamjson-1.0.3.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+streamjson-1.0.3.dist-info/top_level.txt,sha256=_nCeM-g6_juctn1iAfz9vGbuKPFrqB-VH85dI5uHlRU,17
+streamjson-1.0.3.dist-info/RECORD,,

streamjson-1.0.3.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

streamjson-1.0.3.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zairon Jacobs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

streamjson-1.0.3.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+streamjson
+tests

tests/test_app.py

@@ -0,0 +1,59 @@
+import os
+import json
+
+import pytest
+
+from streamjson import StreamJSONWriter, StreamJSONReader
+
+tmp_json_file = os.path.join(os.path.dirname(__file__), "test.json")
+
+
+class TestStreamJSON:
+    def test_writer_send(self):
+        persons = [
+            {"id": "0001", "first_name": "John", "last_name": "Doe"},
+            {"id": "0002", "first_name": "Jane", "last_name": "Doe"},
+        ]
+
+        with StreamJSONWriter(tmp_json_file, indent=2) as writer:
+            for person in persons:
+                writer.send(person)
+
+        with open(tmp_json_file, "r") as file:
+            expected = persons
+            result = json.load(file)
+
+            assert expected == result
+
+    def test_reader_find(self):
+        persons_json_file = os.path.join(os.path.dirname(__file__), "persons.json")
+        with StreamJSONReader(persons_json_file) as reader:
+            reader_generator = reader.find()
+            obj_1 = next(reader_generator)
+            obj_2 = next(reader_generator)
+
+            assert obj_1 == {
+                "id": "0001",
+                "first_name": "John",
+                "last_name": "Doe",
+                "hobbies": [
+                    {"name": "photography"},
+                    {"name": "programming"},
+                    {"name": "hiking"},
+                ],
+            } and obj_2 == {
+                "id": "0002",
+                "first_name": "Jane",
+                "last_name": "Doe",
+                "hobbies": [{"name": "dancing"}],
+            }
+
+    @pytest.fixture(scope="session", autouse=True)
+    def cleanup(self):
+        """
+        Delete file after tests are finished.
+        """
+
+        yield
+        if os.path.exists(tmp_json_file):
+            os.remove(tmp_json_file)