edq-utils 0.1.9__py3-none-any.whl

edq/util/encoding.py

@@ -0,0 +1,18 @@
+import base64
+import typing
+
+import edq.util.dirent
+
+def to_base64(data: typing.Union[bytes, str], encoding: str = edq.util.dirent.DEFAULT_ENCODING) -> str:
+    """ Convert a payload to a base64-encoded string. """
+
+    if (isinstance(data, str)):
+        data = data.encode(encoding)
+
+    data = base64.standard_b64encode(data)
+    return data.decode(encoding)
+
+def from_base64(data: str, encoding: str = edq.util.dirent.DEFAULT_ENCODING) -> bytes:
+    """ Convert a base64-encoded string to bytes. """
+
+    return base64.b64decode(data.encode(encoding), validate = True)

edq/util/hash.py

@@ -0,0 +1,41 @@
+import hashlib
+import typing
+
+import edq.util.dirent
+
+DEFAULT_CLIP_HASH_LENGTH: int = 8
+
+def sha256_hex(payload: typing.Any, encoding: str = edq.util.dirent.DEFAULT_ENCODING) -> str:
+    """ Compute and return the hex string of the SHA3-256 encoding of the payload. """
+
+    if (isinstance(payload, str)):
+        payload = payload.encode(encoding)
+
+    digest = hashlib.new('sha256')
+    digest.update(payload)
+    return digest.hexdigest()
+
+def clip_text(text: str, max_length: int, hash_length: int = DEFAULT_CLIP_HASH_LENGTH) -> str:
+    """
+    Return a clipped version of the input text that is no longer than the specified length.
+    If the base text is found to be too long,
+    then enough if the tail of the text will be removed to insert a note about the clipping
+    and the first |hash_length| characters of the hash from sha256_hex().
+
+    Note that the max length is actually a soft cap.
+    Longer strings can be generated if the original text is shorter than the notification
+    that will be inserted into the clipped text.
+    """
+
+    if (len(text) <= max_length):
+        return text
+
+    hash_hex = sha256_hex(text)
+    notification = f"[text clipped {hash_hex[0:hash_length]}]"
+
+    # Don't clip the text if the final string would be longer.
+    if (len(notification) >= len(text)):
+        return text
+
+    keep_length = max(0, max_length - len(notification))
+    return text[0:keep_length] + notification

edq/util/hash_test.py

@@ -0,0 +1,89 @@
+import edq.testing.unittest
+import edq.util.hash
+
+class TestHash(edq.testing.unittest.BaseTest):
+    """ Test hash-based operations. """
+
+    def test_sha256_hex_base(self):
+        """ Test the base sha256 hash. """
+
+        # [(input, expected), ...]
+        test_cases = [
+            ('foo', '2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae'),
+            ('abcdefghijklmnopqrstuvwxyz1234567890', '77d721c817f9d216c1fb783bcad9cdc20aaa2427402683f1f75dd6dfbe657470'),
+            ('', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'),
+        ]
+
+        for (i, test_case) in enumerate(test_cases):
+            (text, expected) = test_case
+
+            with self.subTest(msg = f"Case {i} ('{text}'):"):
+                actual = edq.util.hash.sha256_hex(text)
+                self.assertEqual(expected, actual)
+
+    def test_clip_text_base(self):
+        """ Test the base functionality of clip_text(). """
+
+        # [(text, max length, kwargs, expected), ...]
+        test_cases = [
+            # No Clip
+            (
+                'abcdefghijklmnopqrstuvwxyz1234567890',
+                100,
+                {},
+                'abcdefghijklmnopqrstuvwxyz1234567890',
+            ),
+
+            # Base Clip
+            (
+                'abcdefghijklmnopqrstuvwxyz1234567890',
+                30,
+                {},
+                'abcdefg[text clipped 77d721c8]',
+            ),
+
+            # Full Clip
+            (
+                'abcdefghijklmnopqrstuvwxyz1234567890',
+                23,
+                {},
+                '[text clipped 77d721c8]',
+            ),
+
+            # Over Clip
+            (
+                'abcdefghijklmnopqrstuvwxyz1234567890',
+                10,
+                {},
+                '[text clipped 77d721c8]',
+            ),
+
+            # Different Hash Length
+            (
+                'abcdefghijklmnopqrstuvwxyz1234567890',
+                30,
+                {'hash_length': 10},
+                'abcde[text clipped 77d721c817]',
+            ),
+
+            # Notification Longer Than Text
+            (
+                'abc',
+                1,
+                {},
+                'abc',
+            ),
+            (
+                'abcdefghijklmnopqrstuvwxyz1234567890',
+                10,
+                {'hash_length': 64},
+                'abcdefghijklmnopqrstuvwxyz1234567890',
+            ),
+        ]
+
+        for (i, test_case) in enumerate(test_cases):
+            (text, max_length, kwargs, expected) = test_case
+
+            with self.subTest(msg = f"Case {i} ('{text}'):"):
+                actual = edq.util.hash.clip_text(text, max_length, **kwargs)
+                self.assertEqual(expected, actual)

edq/util/json.py

@@ -0,0 +1,180 @@
+"""
+This file standardizes how we write and read JSON.
+Specifically, we try to be flexible when reading (using JSON5),
+and strict when writing (using vanilla JSON).
+"""
+
+import enum
+import json
+import os
+import typing
+
+import json5
+
+import edq.util.dirent
+
+class DictConverter():
+    """
+    A base class for class that can represent (serialize) and reconstruct (deserialize) themselves as/from a dict.
+    The intention is that the dict can then be cleanly converted to/from JSON.
+
+    General (but inefficient) implementations of several core Python equality, comparison, and representation methods are provided.
+    """
+
+    def to_dict(self) -> typing.Dict[str, typing.Any]:
+        """
+        Return a dict that can be used to represent this object.
+        If the dict is passed to from_dict(), an identical object should be reconstructed.
+
+        A general (but inefficient) implementation is provided by default.
+        """
+
+        return vars(self).copy()
+
+    @classmethod
+    # Note that `typing.Self` is returned, but that is introduced in Python 3.12.
+    def from_dict(cls, data: typing.Dict[str, typing.Any]) -> typing.Any:
+        """
+        Return an instance of this subclass created using the given dict.
+        If the dict came from to_dict(), the returned object should be identical to the original.
+
+        A general (but inefficient) implementation is provided by default.
+        """
+
+        return cls(**data)
+
+    def __eq__(self, other: object) -> bool:
+        """
+        Check for equality.
+
+        This check uses to_dict() and compares the results.
+        This may not be complete or efficient depending on the child class.
+        """
+
+        # Note the hard type check (done so we can keep this method general).
+        if (type(self) != type(other)):  # pylint: disable=unidiomatic-typecheck
+            return False
+
+        return bool(self.to_dict() == other.to_dict())  # type: ignore[attr-defined]
+
+    def __lt__(self, other: 'DictConverter') -> bool:
+        return dumps(self) < dumps(other)
+
+    def __hash__(self) -> int:
+        return hash(dumps(self))
+
+    def __str__(self) -> str:
+        return dumps(self)
+
+    def __repr__(self) -> str:
+        return dumps(self)
+
+def _custom_handle(value: typing.Any) -> typing.Union[typing.Dict[str, typing.Any], str]:
+    """
+    Handle objects that are not JSON serializable by default,
+    e.g., calling vars() on an object.
+    """
+
+    if (isinstance(value, DictConverter)):
+        return value.to_dict()
+
+    if (isinstance(value, enum.Enum)):
+        return str(value)
+
+    if (hasattr(value, '__dict__')):
+        return dict(vars(value))
+
+    raise ValueError(f"Could not JSON serialize object: '{value}'.")
+
+def load(file_obj: typing.TextIO, strict: bool = False, **kwargs: typing.Any) -> typing.Any:
+    """
+    Load a file object/handler as JSON.
+    If strict is set, then use standard Python JSON,
+    otherwise use JSON5.
+    """
+
+    if (strict):
+        return json.load(file_obj, **kwargs)
+
+    return json5.load(file_obj, **kwargs)
+
+def loads(text: str, strict: bool = False, **kwargs: typing.Any) -> typing.Any:
+    """
+    Load a string as JSON.
+    If strict is set, then use standard Python JSON,
+    otherwise use JSON5.
+    """
+
+    if (strict):
+        return json.loads(text, **kwargs)
+
+    return json5.loads(text, **kwargs)
+
+def load_path(
+        path: str,
+        strict: bool = False,
+        encoding: str = edq.util.dirent.DEFAULT_ENCODING,
+        **kwargs: typing.Any) -> typing.Any:
+    """
+    Load a file path as JSON.
+    If strict is set, then use standard Python JSON,
+    otherwise use JSON5.
+    """
+
+    if (os.path.isdir(path)):
+        raise IsADirectoryError(f"Cannot open JSON file, expected a file but got a directory at '{path}'.")
+
+    try:
+        with open(path, 'r', encoding = encoding) as file:
+            return load(file, strict = strict, **kwargs)
+    except Exception as ex:
+        raise ValueError(f"Failed to read JSON file '{path}'.") from ex
+
+def loads_object(text: str, cls: typing.Type[DictConverter], **kwargs: typing.Any) -> DictConverter:
+    """ Load a JSON string into an object (which is a subclass of DictConverter). """
+
+    data = loads(text, **kwargs)
+    if (not isinstance(data, dict)):
+        raise ValueError(f"JSON to load into an object is not a dict, found '{type(data)}'.")
+
+    return cls.from_dict(data)  # type: ignore[no-any-return]
+
+def load_object_path(path: str, cls: typing.Type[DictConverter], **kwargs: typing.Any) -> DictConverter:
+    """ Load a JSON file into an object (which is a subclass of DictConverter). """
+
+    data = load_path(path, **kwargs)
+    if (not isinstance(data, dict)):
+        raise ValueError(f"JSON to load into an object is not a dict, found '{type(data)}'.")
+
+    return cls.from_dict(data)  # type: ignore[no-any-return]
+
+def dump(
+        data: typing.Any,
+        file_obj: typing.TextIO,
+        default: typing.Union[typing.Callable, None] = _custom_handle,
+        sort_keys: bool = True,
+        **kwargs: typing.Any) -> None:
+    """ Dump an object as a JSON file object. """
+
+    json.dump(data, file_obj, default = default, sort_keys = sort_keys, **kwargs)
+
+def dumps(
+        data: typing.Any,
+        default: typing.Union[typing.Callable, None] = _custom_handle,
+        sort_keys: bool = True,
+        **kwargs: typing.Any) -> str:
+    """ Dump an object as a JSON string. """
+
+    return json.dumps(data, default = default, sort_keys = sort_keys, **kwargs)
+
+def dump_path(
+        data: typing.Any,
+        path: str,
+        default: typing.Union[typing.Callable, None] = _custom_handle,
+        sort_keys: bool = True,
+        encoding: str = edq.util.dirent.DEFAULT_ENCODING,
+        **kwargs: typing.Any) -> None:
+    """ Dump an object as a JSON file. """
+
+    with open(path, 'w', encoding = encoding) as file:
+        dump(data, file, default = default, sort_keys = sort_keys, **kwargs)

edq/util/json_test.py

@@ -0,0 +1,228 @@
+import os
+import typing
+
+import edq.testing.unittest
+import edq.util.dirent
+import edq.util.json
+import edq.util.reflection
+
+class TestJSON(edq.testing.unittest.BaseTest):
+    """ Test JSON utils. """
+
+    def test_loading_dumping_base(self):
+        """
+        Test the family of JSON loading and dumping functions.
+        """
+
+        # [(string, dict, strict?, error_substring), ...]
+        test_cases = [
+            # Base
+            (
+                '{"a": 1}',
+                {"a": 1},
+                False,
+                None,
+            ),
+
+            # Trivial - Strict
+            (
+                '{"a": 1}',
+                {"a": 1},
+                True,
+                None,
+            ),
+
+            # JSON5
+            (
+                '{"a": 1,}',
+                {"a": 1},
+                False,
+                None,
+            ),
+
+            # JSON5 - Strict
+            (
+                '{"a": 1,}',
+                {"a": 1},
+                True,
+                'JSONDecodeError',
+            ),
+        ]
+
+        # [(function, name), ...]
+        test_methods = [
+            (self._subtest_loads_dumps, 'subtest_loads_dumps'),
+            (self._subtest_load_dump, 'subtest_load_dump'),
+            (self._subtest_load_dump_path, 'subtest_load_dump_path'),
+        ]
+
+        for (test_method, test_method_name) in test_methods:
+            for (i, test_case) in enumerate(test_cases):
+                (text_content, dict_content, strict, error_substring) = test_case
+
+                with self.subTest(msg = f"Subtest {test_method_name}, Case {i} ('{text_content}'):"):
+                    try:
+                        test_method(text_content, dict_content, strict)
+                    except AssertionError:
+                        # The subttest failed an assertion.
+                        raise
+                    except Exception as ex:
+                        error_string = self.format_error_string(ex)
+                        if (error_substring is None):
+                            self.fail(f"Unexpected error: '{error_string}'.")
+
+                        self.assertIn(error_substring, error_string, 'Error is not as expected.')
+
+                        continue
+
+                    if (error_substring is not None):
+                        self.fail(f"Did not get expected error: '{error_substring}'.")
+
+    def _subtest_loads_dumps(self, text_content, dict_content, strict):
+        actual_dict = edq.util.json.loads(text_content, strict = strict)
+        actual_text = edq.util.json.dumps(dict_content)
+        double_conversion_text = edq.util.json.dumps(actual_dict)
+
+        self.assertDictEqual(dict_content, actual_dict)
+        self.assertEqual(actual_text, double_conversion_text)
+
+    def _subtest_load_dump(self, text_content, dict_content, strict):
+        temp_dir = edq.util.dirent.get_temp_dir(prefix = 'edq_test_json_')
+
+        path_text = os.path.join(temp_dir, 'test-text.json')
+        path_dict = os.path.join(temp_dir, 'test-dict.json')
+
+        edq.util.dirent.write_file(path_text, text_content)
+
+        with open(path_text, 'r', encoding = edq.util.dirent.DEFAULT_ENCODING) as file:
+            text_load = edq.util.json.load(file, strict = strict)
+
+        with open(path_dict, 'w', encoding = edq.util.dirent.DEFAULT_ENCODING) as file:
+            edq.util.json.dump(dict_content, file)
+
+        with open(path_dict, 'r', encoding = edq.util.dirent.DEFAULT_ENCODING) as file:
+            dict_load = edq.util.json.load(file, strict = strict)
+
+        self.assertDictEqual(dict_content, text_load)
+        self.assertDictEqual(dict_load, text_load)
+
+    def _subtest_load_dump_path(self, text_content, dict_content, strict):
+        temp_dir = edq.util.dirent.get_temp_dir(prefix = 'edq_test_json_path_')
+
+        path_text = os.path.join(temp_dir, 'test-text.json')
+        path_dict = os.path.join(temp_dir, 'test-dict.json')
+
+        edq.util.dirent.write_file(path_text, text_content)
+        text_load = edq.util.json.load_path(path_text, strict = strict)
+
+        edq.util.json.dump_path(dict_content, path_dict)
+        dict_load = edq.util.json.load_path(path_dict, strict = strict)
+
+        self.assertDictEqual(dict_content, text_load)
+        self.assertDictEqual(dict_load, text_load)
+
+    def test_object_base(self):
+        """
+        Test loading and dumping JSON objects
+        """
+
+        # [(string, object, error_substring), ...]
+        test_cases = [
+            # Base
+            (
+                '{"a": 1, "b": "b"}',
+                _TestConverter(1, "b"),
+                None,
+            ),
+
+            # Missing Key
+            (
+                '{"a": 1}',
+                _TestConverter(1, None),
+                None,
+            ),
+
+            # Empty
+            (
+                '{}',
+                _TestConverter(None, None),
+                None,
+            ),
+
+            # Extra Key
+            (
+                '{"a": 1, "b": "b", "c": 0}',
+                _TestConverter(1, "b"),
+                None,
+            ),
+
+            # List
+            (
+                '[{"a": 1, "b": "b"}]',
+                _TestConverter(1, "b"),
+                'not a dict',
+            ),
+        ]
+
+        # [(function, name), ...]
+        test_methods = [
+            (self._subtest_loads_object, 'subtest_loads_object'),
+            (self._subtest_load_object_path, 'subtest_load_object_path'),
+        ]
+
+        for (test_method, test_method_name) in test_methods:
+            for (i, test_case) in enumerate(test_cases):
+                (text_content, object_content, error_substring) = test_case
+
+                with self.subTest(msg = f"Subtest {test_method_name}, Case {i} ('{text_content}'):"):
+                    try:
+                        test_method(text_content, object_content)
+                    except AssertionError:
+                        # The subttest failed an assertion.
+                        raise
+                    except Exception as ex:
+                        error_string = self.format_error_string(ex)
+                        if (error_substring is None):
+                            self.fail(f"Unexpected error: '{error_string}'.")
+
+                        self.assertIn(error_substring, error_string, 'Error is not as expected.')
+
+                        continue
+
+                    if (error_substring is not None):
+                        self.fail(f"Did not get expected error: '{error_substring}'.")
+
+    def _subtest_loads_object(self, text_content, object_content):
+        actual_object = edq.util.json.loads_object(text_content, _TestConverter)
+        actual_text = edq.util.json.dumps(object_content)
+        double_conversion_text = edq.util.json.dumps(actual_object)
+
+        self.assertEqual(object_content, actual_object)
+        self.assertEqual(actual_text, double_conversion_text)
+
+    def _subtest_load_object_path(self, text_content, object_content):
+        temp_dir = edq.util.dirent.get_temp_dir(prefix = 'edq_test_json_object_path_')
+
+        path_text = os.path.join(temp_dir, 'test-text.json')
+        path_object = os.path.join(temp_dir, 'test-object.json')
+
+        edq.util.dirent.write_file(path_text, text_content)
+        text_load = edq.util.json.load_object_path(path_text, _TestConverter)
+
+        edq.util.json.dump_path(object_content, path_object)
+        object_load = edq.util.json.load_object_path(path_object, _TestConverter)
+
+        self.assertEqual(object_content, text_load)
+        self.assertEqual(object_load, text_load)
+
+class _TestConverter(edq.util.json.DictConverter):
+    def __init__(self, a: typing.Union[int, None] = None, b: typing.Union[str, None] = None, **kwargs) -> None:
+        self.a: typing.Union[int, None] = a
+        self.b: typing.Union[str, None] = b
+
+    def to_dict(self) -> typing.Dict[str, typing.Any]:
+        return vars(self)
+
+    @classmethod
+    def from_dict(cls, data: typing.Dict[str, typing.Any]) -> typing.Any:
+        return _TestConverter(**data)