compress-json 1.1.2__py3-none-any.whl

compress_json/__init__.py

@@ -0,0 +1,11 @@
+"""Compress JSON is a package to handle reading and writing of compressed JSON documents."""
+
+from compress_json.compress_json import (
+    load,
+    dump,
+    local_load,
+    local_dump,
+    get_supported_extensions,
+)
+
+__all__ = ["load", "dump", "local_load", "local_dump", "get_supported_extensions"]

compress_json/__version__.py

@@ -0,0 +1,3 @@
+"""Current version of package compress_json."""
+
+__version__ = "1.1.2"

compress_json/compress_json.py

@@ -0,0 +1,316 @@
+# -*- coding: utf-8 -*-
+"""
+A thin wrapper of standard ``json`` with standard compression libraries.
+"""
+import json
+from typing import Dict, Any, Optional, List
+import traceback
+import os
+
+__all__ = ["dump", "load", "local_dump", "local_load"]
+
+_DEFAULT_EXTENSION_MAP = {
+    "json": "json",
+    "gz": "gzip",
+    "bz": "bz2",
+    "lzma": "lzma",
+    "xz": "lzma",
+}
+
+_DEFAULT_COMPRESSION_WRITE_MODES = {
+    "json": "w",
+    "gzip": "wt",
+    "bz2": "wt",
+    "lzma": "wt",
+}
+
+_DEFAULT_COMPRESSION_READ_MODES = {"json": "r", "gzip": "rt", "bz2": "rt", "lzma": "rt"}
+
+_CACHE: Dict = {}
+
+
+def get_supported_extensions() -> List[str]:
+    """Return the supported extensions for compression algorithms."""
+    return list(_DEFAULT_EXTENSION_MAP.keys())
+
+
+def get_compression_write_mode(compression: str) -> str:
+    """Return mode for opening file buffer for writing.
+
+    Parameters
+    -------------------
+    compression: str
+        The extension of the compression to be used.
+
+    Returns
+    -------------------
+    The code to use for opening the file in write mode.
+    """
+    return _DEFAULT_COMPRESSION_WRITE_MODES[compression]
+
+
+def get_compression_read_mode(compression: str) -> str:
+    """Return mode for opening file buffer for reading.
+
+    Parameters
+    -------------------
+    compression: str
+        The extension of the compression to be used.
+
+    Returns
+    -------------------
+    The code to use for opening the file in read mode.
+    """
+    return _DEFAULT_COMPRESSION_READ_MODES[compression]
+
+
+def infer_compression_from_filename(filename: str) -> str:
+    """Return the compression protocal inferred from given filename.
+
+    Parameters
+    ----------
+    filename: str
+        The filename for which to infer the compression protocol
+    """
+    return _DEFAULT_EXTENSION_MAP[filename.split(".")[-1]]
+
+
+def dump(
+    obj: Any,
+    path: str,
+    compression: Optional[str] = None,
+    compression_kwargs: Optional[Dict] = None,
+    json_kwargs: Optional[Dict] = None,
+    encoding: str = "utf-8",
+):
+    """Dump the contents of an object to disk as json using the detected compression protocol.
+
+    Parameters
+    ----------------
+    obj: any
+        The object that will be saved to disk
+    path: str
+        The path to the file to which to dump ``obj``
+    compression: Optional[str] = None
+        The compression protocol to use. If None, it will be inferred from the file extension.
+    compression_kwargs: Optional[Dict] = None
+        Keywords argument to pass to the compressed file opening protocol.
+    json_kwargs: Optional[Dict] = None
+        Keywords argument to pass to the json file opening protocol.
+    encoding: str = "utf-8"
+        The encoding to use to dump the document. By default, UTF8.
+
+    Raises
+    ----------------
+    ValueError
+        If given path is not a valid string.
+    """
+    if not isinstance(path, str):
+        if isinstance(obj, str):
+            raise ValueError(
+                (
+                    "The object you have provided to the dump method is a string "
+                    "while the object you have provided as a path is NOT a string "
+                    f"but an object of type {type(path)}. Maybe you need to swap them?"
+                )
+            )
+        raise ValueError("The given path is not a string.")
+
+    compression_kwargs = {} if compression_kwargs is None else compression_kwargs
+    json_kwargs = {} if json_kwargs is None else json_kwargs
+    compression = (
+        infer_compression_from_filename(path) if compression is None else compression
+    )
+    mode = get_compression_write_mode(compression)
+
+    directory = os.path.dirname(path)
+    if directory:
+        os.makedirs(directory, exist_ok=True)
+
+    if compression is None or compression == "json":
+        fout = open(path, mode=mode, encoding=encoding, **compression_kwargs)
+    elif compression == "gzip":
+        import gzip  # pylint: disable=import-outside-toplevel
+
+        fout = gzip.open(path, mode=mode, encoding=encoding, **compression_kwargs)  # type: ignore
+    elif compression == "bz2":
+        import bz2  # pylint: disable=import-outside-toplevel
+
+        fout = bz2.open(path, mode=mode, encoding=encoding, **compression_kwargs)  # type: ignore
+    elif compression == "lzma":
+        import lzma  # pylint: disable=import-outside-toplevel
+
+        fout = lzma.open(path, mode=mode, encoding=encoding, **compression_kwargs)  # type: ignore
+    else:
+        raise ValueError(f"Unsupported compression protocol: {compression}")
+
+    with fout:
+        json.dump(obj, fout, **json_kwargs)
+
+
+def load(
+    path: str,
+    compression: Optional[str] = None,
+    compression_kwargs: Optional[Dict] = None,
+    json_kwargs: Optional[Dict] = None,
+    encoding: str = "utf-8",
+    use_cache: bool = False,
+):
+    """Return json object at given path uncompressed with detected compression protocol.
+
+    Parameters
+    ----------
+    path: str
+        The path to the file from which to load the ``obj``
+    compression: Optional[str] = None
+        The compression protocol to use. If None, it will be inferred from the file extension.
+    compression_kwargs: Optional[Dict] = None
+        Keywords argument to pass to the compressed file opening protocol.
+    json_kwargs: Optional[Dict] = None
+        Keywords argument to pass to the json file opening protocol.
+    encoding: str = "utf-8"
+        The encoding to use to load the document. By default, UTF8.
+    use_cache: bool = False
+        Whether to put loaded JSON files in a static cache object.
+
+    Raises
+    ----------------
+    ValueError
+        If given path is not a valid string.
+    """
+    if not isinstance(path, str):
+        raise ValueError("The given path is not a string.")
+
+    if use_cache and path in _CACHE:
+        return _CACHE[path]
+
+    compression_kwargs = {} if compression_kwargs is None else compression_kwargs
+    json_kwargs = {} if json_kwargs is None else json_kwargs
+    compression = (
+        infer_compression_from_filename(path) if compression is None else compression
+    )
+    mode = get_compression_read_mode(compression)
+
+    if compression is None or compression == "json":
+        file = open(path, mode=mode, encoding=encoding, **compression_kwargs)
+    elif compression == "gzip":
+        import gzip  # pylint: disable=import-outside-toplevel
+
+        file = gzip.open(path, mode=mode, encoding=encoding, **compression_kwargs)  # type: ignore
+    elif compression == "bz2":
+        import bz2  # pylint: disable=import-outside-toplevel
+
+        file = bz2.open(path, mode=mode, encoding=encoding, **compression_kwargs)
+    elif compression == "lzma":
+        import lzma  # pylint: disable=import-outside-toplevel
+
+        file = lzma.open(path, mode=mode, encoding=encoding, **compression_kwargs)
+    else:
+        raise ValueError(f"Unsupported compression protocol: {compression}")
+
+    with file:
+        json_content = json.load(file, **json_kwargs)
+
+    if use_cache:
+        _CACHE[path] = json_content
+
+    return json_content
+
+
+def local_path(relative_path: str) -> str:
+    """Return path localized to caller function.
+
+    Parameters
+    -----------------------
+    relative_path: str
+        The path to be made absolute to the caller function.
+
+    Returns
+    -----------------------
+    The absolute path with as root the caller function.
+    """
+    return os.path.join(
+        os.path.dirname(os.path.abspath(traceback.extract_stack()[-3].filename)),
+        relative_path,
+    )
+
+
+def local_load(
+    path: str,
+    compression: Optional[str] = None,
+    compression_kwargs: Optional[Dict] = None,
+    json_kwargs: Optional[Dict] = None,
+    encoding: str = "utf-8",
+    use_cache: bool = False,
+) -> Any:
+    """Return json object at given local path uncompressed with detected compression protocol.
+
+    Parameters
+    ----------
+    path: str
+        The path to the local file from which to load the ``obj``
+    compression: Optional[str] = None
+        The compression protocol to use. If None, it will be inferred from the file extension.
+    compression_kwargs: Optional[Dict] = None
+        keywords argument to pass to the compressed file opening protocol.
+    json_kwargs: Optional[Dict] = None
+        keywords argument to pass to the json file opening protocol.
+    encoding: str = "utf-8"
+        The encoding to use to load the document. By default, UTF8.
+    use_cache: bool = False
+        Whether to put loaded JSON files in a static cache object.
+
+    Raises
+    ----------------
+    ValueError,
+        If given path is not a valid string.
+    """
+    return load(
+        path=local_path(path),
+        compression=compression,
+        compression_kwargs=compression_kwargs,
+        json_kwargs=json_kwargs,
+        encoding=encoding,
+        use_cache=use_cache,
+    )
+
+
+def local_dump(
+    obj: Any,
+    path: str,
+    compression: Optional[str] = None,
+    compression_kwargs: Optional[Dict] = None,
+    json_kwargs: Optional[Dict] = None,
+    encoding: str = "utf-8",
+):
+    """Dump the contents of an object to disk as json, using the detected compression protocol.
+
+    Parameters
+    ----------
+    obj: Any
+        The object that will be saved to disk
+    compression: Optional[str] = None
+        The compression protocol to use. If None, it will be inferred from the file extension.
+    path: str
+        The local path to the file to which to dump ``obj``
+    compression_kwargs: Optional[Dict] = None
+        keywords argument to pass to the compressed file opening protocol.
+    json_kwargs: Optional[Dict] = None
+        keywords argument to pass to the json file opening protocol.
+    encoding: str = "utf-8"
+        The encoding to use to dump the document. By default, UTF8.
+
+
+    Raises
+    ----------------
+    ValueError
+        If given path is not a valid string.
+    """
+    dump(
+        obj,
+        path=local_path(path),
+        compression=compression,
+        compression_kwargs=compression_kwargs,
+        json_kwargs=json_kwargs,
+        encoding=encoding,
+    )

compress_json-1.1.2.dist-info/METADATA

@@ -0,0 +1,184 @@
+Metadata-Version: 2.4
+Name: compress_json
+Version: 1.1.2
+Summary: The missing Python utility to read and write large compressed JSONs.
+Author-email: Luca Cappelletti <cappelletti.luca94@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/LucaCappelletti94/compress_json
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-readme; extra == "test"
+Requires-Dist: validate_version_code; extra == "test"
+Requires-Dist: random_dict; extra == "test"
+Requires-Dist: dict_hash; extra == "test"
+Dynamic: license-file
+
+# compress_json
+
+[![pip](https://badge.fury.io/py/compress-json.svg)](https://pypi.org/project/compress-json/)
+[![python](https://img.shields.io/pypi/pyversions/compress-json.svg)](https://pypi.org/project/compress-json/)
+[![license](https://img.shields.io/pypi/l/compress-json.svg)](https://pypi.org/project/compress-json/)
+[![downloads](https://pepy.tech/badge/compress-json)](https://pepy.tech/project/compress-json)
+[![Github Actions](https://github.com/LucaCappelletti94/compress_json/actions/workflows/python.yml/badge.svg)](https://github.com/LucaCappelletti94/compress_json/actions/workflows/python.yml)
+
+The missing Python utility to read and write compressed JSONs.
+
+The library is loosely based on the [compress_pickle](https://github.com/lucianopaz/compress_pickle) library.
+
+## How do I install this package?
+
+As usual, just download it using pip:
+
+```shell
+pip install compress_json
+```
+
+## Available compression modes
+
+The compression modes, detected automatically by the file name, are **gzip**, **bz2**, and **lzma** (or **xz**), with the notable exception of **zip** which seems difficult to integrate into the JSON pipeline.
+
+## Usage example
+
+The library is extremely easy to use:
+
+```python
+import compress_json
+
+D = {
+    "A": {
+        "B": "C"
+    }
+}
+compress_json.dump(D, "filepath.json.gz")   # for a gzip file
+compress_json.dump(D, "filepath.json.bz")   # for a bz2 file
+compress_json.dump(D, "filepath.json.lzma") # for a lzma file
+compress_json.dump(D, "filepath.json.xz") # for a lzma file
+
+D1 = compress_json.load("filepath.json.gz")   # for loading a gzip file
+D2 = compress_json.load("filepath.json.bz")   # for loading a bz2 file
+D3 = compress_json.load("filepath.json.lzma") # for loading a lzma file
+D3 = compress_json.load("filepath.json.xz") # for loading a lzma file
+```
+
+If it happens that you have to load or dump a JSON object with a custom extension, you can specify the compression mode by passing the `compression` parameter to the `load` and `dump` methods:
+
+```python
+import compress_json
+
+D = {
+    "A": {
+        "B": "C"
+    }
+}
+
+compress_json.dump(D, "filepath.custom_extension1", compression="gzip")   # for a gzip file
+compress_json.dump(D, "filepath.custom_extension2", compression="bz2")   # for a bz2 file
+compress_json.dump(D, "filepath.custom_extension3", compression="lzma") # for a lzma file
+
+D1 = compress_json.load("filepath.custom_extension1", compression="gzip")   # for loading a gzip file
+D2 = compress_json.load("filepath.custom_extension2", compression="bz2")   # for loading a bz2 file
+D3 = compress_json.load("filepath.custom_extension3", compression="lzma") # for loading a lzma file
+
+assert D == D1 == D2 == D3
+```
+
+## Some extra perks: local loading and dumping
+
+The library makes available, other than the usual `load` and `dump` from the JSON library, the methods `local_load` and `local_dump`, which let you load and dump files in the same directory as wherever you are calling them, by using the call stack.
+
+This can be useful, especially when loading files within packages.
+
+```python
+import compress_json
+
+D = {
+    "A": {
+        "B": "C"
+    }
+}
+compress_json.local_dump(D, "filepath.json.gz")   # for a gzip file
+compress_json.local_dump(D, "filepath.json.bz")   # for a bz2 file
+compress_json.local_dump(D, "filepath.json.lzma") # for a lzma file
+
+D1 = compress_json.local_load("filepath.json.gz")   # for loading a gzip file
+D2 = compress_json.local_load("filepath.json.bz")   # for loading a bz2 file
+D3 = compress_json.local_load("filepath.json.lzma") # for loading a lzma file
+
+assert D == D1 == D2 == D3
+```
+
+## Loading with RAM cache
+
+Sometimes you need to load a compressed JSON file a LOT of times, and you may want to put this document in a cache or something of the sort. Fortunately, we already provide this option for you:
+
+```python
+import compress_json
+
+# The first time you load the file, it will be cached in RAM
+D1 = compress_json.load(
+    "filepath.json.gz",
+    use_cache=True
+)
+
+# The second time you load the file, it will be loaded from the cache
+D2 = compress_json.local_load(
+    "filepath.json.gz",
+    use_cache=True
+)
+
+assert D1 == D2
+```
+
+## Advanced usage
+
+You can pass parameters to either the chosen compression mode or the JSON library.
+
+With the `json_kwargs` parameter, you can specify any of the kwargs that should be forwarded to the JSON library method, which you can obtain for your Python version by running `help(json.dump)` and `help(json.load)`, depending on whether you are dumping or loading the JSON object.
+
+Similarly, with the `compression_kwargs` parameter, you can specify any parameter that has to be forwarded to the compression library that you intend to use, whether that is `lzma`, `gzip`, or `bz2`, and as per JSON will depend on which version you have installed.
+
+Whether you are dumping or loading a compressed JSON object, you can get the list of parameters you have available to forward to the compression method by running `help(lzma.open)`, `help(gzip.open)`, or `help(bz2.open)`, respectively.
+
+```python
+import compress_json
+
+D = {
+    "A": {
+        "B": "C"
+    }
+}
+compress_json.dump(
+    D, "filepath.json.gz",
+    compression_kwargs={
+        "compresslevel": 9 # The kwargs for gzip
+    },
+    json_kwargs={
+        "indent": 4 # The kwargs for json
+    }
+)
+
+D4 = compress_json.load(
+    "filepath.json.gz",
+    compression_kwargs={ 
+        "compresslevel": 9 # The kwargs for gzip
+    },
+    json_kwargs={} # The kwargs for json
+)
+
+assert D == D4
+```
+
+## Building
+
+Run `uv build` to create the source distribution and wheel.
+
+## Releasing
+
+Set a new version in `compress_json/__version__.py` and publish a GitHub release with a matching tag, optionally prefixed with `v`. The `publish.yml` workflow tests, builds, and publishes the source distribution and wheel through the `pypi` environment using PyPI Trusted Publishing.
+
+## License
+The library is released under the MIT license.

compress_json-1.1.2.dist-info/RECORD

@@ -0,0 +1,9 @@
+compress_json/__init__.py,sha256=fhtJ3CZ40fPT343lz_h6K6KF9S2-wKfr4aA3HK1RQz8,304
+compress_json/__version__.py,sha256=wn6hBWafD884d7BvsdlIOHVc27UQf_WvQ77c5UpE3tY,71
+compress_json/compress_json.py,sha256=BvgbWOidcWvKRezFttMWSVJ9ZAJp1PsO2VY1rSkwLGs,10029
+compress_json/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+compress_json-1.1.2.dist-info/licenses/LICENSE,sha256=kVjMPCxT2u-df_moY6ewQUZXHsUMDbmFxD0rNRCiV0A,1073
+compress_json-1.1.2.dist-info/METADATA,sha256=8q1fQDaFvW5j0Yp93TywqyNSYiUsPimD5hklAoGs4No,6670
+compress_json-1.1.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+compress_json-1.1.2.dist-info/top_level.txt,sha256=ZnXQYQDDh0upVcLx2g52dIBtii99Flgj5yin--JgmzI,14
+compress_json-1.1.2.dist-info/RECORD,,

compress_json-1.1.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

compress_json-1.1.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Luca Cappelletti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

compress_json-1.1.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+compress_json