edq-utils 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

edq/util/json.py

@@ -0,0 +1,163 @@
+"""
+This file standardizes how we write and read JSON.
+Specifically, we try to be flexible when reading (using JSON5),
+and strict when writing (using vanilla JSON).
+"""
+
+import abc
+import enum
+import json
+import typing
+
+import json5
+
+import edq.util.dirent
+
+class DictConverter(abc.ABC):
+    """
+    A base class for class that can represent (serialize) and reconstruct (deserialize) themselves as/from a dict.
+    The intention is that the dict can then be cleanly converted to/from JSON.
+    """
+
+    @abc.abstractmethod
+    def to_dict(self) -> typing.Dict[str, typing.Any]:
+        """
+        Return a dict that can be used to represent this object.
+        If the dict is passed to from_dict(), an identical object should be reconstructed.
+        """
+
+    @classmethod
+    @abc.abstractmethod
+    # Note that `typing.Self` is returned, but that is introduced in Python 3.12.
+    def from_dict(cls, data: typing.Dict[str, typing.Any]) -> typing.Any:
+        """
+        Return an instance of this subclass created using the given dict.
+        If the dict came from to_dict(), the returned object should be identical to the original.
+        """
+
+    def __eq__(self, other: object) -> bool:
+        """
+        Check for equality.
+
+        This check uses to_dict() and compares the results.
+        This may not be complete or efficient depending on the child class.
+        """
+
+        # Note the hard type check (done so we can keep this method general).
+        if (type(self) != type(other)):  # pylint: disable=unidiomatic-typecheck
+            return False
+
+        return self.to_dict() == other.to_dict()  # type: ignore[attr-defined]
+
+    def __str__(self) -> str:
+        return dumps(self)
+
+    def __repr__(self) -> str:
+        return dumps(self)
+
+def _custom_handle(value: typing.Any) -> typing.Union[typing.Dict[str, typing.Any], str]:
+    """
+    Handle objects that are not JSON serializable by default,
+    e.g., calling vars() on an object.
+    """
+
+    if (isinstance(value, DictConverter)):
+        return value.to_dict()
+
+    if (isinstance(value, enum.Enum)):
+        return str(value)
+
+    if (hasattr(value, '__dict__')):
+        return vars(value)
+
+    raise ValueError(f"Could not JSON serialize object: '{value}'.")
+
+def load(file_obj: typing.TextIO, strict: bool = False, **kwargs) -> typing.Dict[str, typing.Any]:
+    """
+    Load a file object/handler as JSON.
+    If strict is set, then use standard Python JSON,
+    otherwise use JSON5.
+    """
+
+    if (strict):
+        return json.load(file_obj, **kwargs)
+
+    return json5.load(file_obj, **kwargs)
+
+def loads(text: str, strict: bool = False, **kwargs) -> typing.Dict[str, typing.Any]:
+    """
+    Load a string as JSON.
+    If strict is set, then use standard Python JSON,
+    otherwise use JSON5.
+    """
+
+    if (strict):
+        return json.loads(text, **kwargs)
+
+    return json5.loads(text, **kwargs)
+
+def load_path(
+        path: str,
+        strict: bool = False,
+        encoding: str = edq.util.dirent.DEFAULT_ENCODING,
+        **kwargs) -> typing.Dict[str, typing.Any]:
+    """
+    Load a file path as JSON.
+    If strict is set, then use standard Python JSON,
+    otherwise use JSON5.
+    """
+
+    try:
+        with open(path, 'r', encoding = encoding) as file:
+            return load(file, strict = strict, **kwargs)
+    except Exception as ex:
+        raise ValueError(f"Failed to read JSON file '{path}'.") from ex
+
+def loads_object(text: str, cls: typing.Type[DictConverter], **kwargs) -> DictConverter:
+    """ Load a JSON string into an object (which is a subclass of DictConverter). """
+
+    data = loads(text, **kwargs)
+    if (not isinstance(data, dict)):
+        raise ValueError(f"JSON to load into an object is not a dict, found '{type(data)}'.")
+
+    return cls.from_dict(data)
+
+def load_object_path(path: str, cls: typing.Type[DictConverter], **kwargs) -> DictConverter:
+    """ Load a JSON file into an object (which is a subclass of DictConverter). """
+
+    data = load_path(path, **kwargs)
+    if (not isinstance(data, dict)):
+        raise ValueError(f"JSON to load into an object is not a dict, found '{type(data)}'.")
+
+    return cls.from_dict(data)
+
+def dump(
+        data: typing.Any,
+        file_obj: typing.TextIO,
+        default: typing.Union[typing.Callable, None] = _custom_handle,
+        sort_keys: bool = True,
+        **kwargs) -> None:
+    """ Dump an object as a JSON file object. """
+
+    json.dump(data, file_obj, default = default, sort_keys = sort_keys, **kwargs)
+
+def dumps(
+        data: typing.Any,
+        default: typing.Union[typing.Callable, None] = _custom_handle,
+        sort_keys: bool = True,
+        **kwargs) -> str:
+    """ Dump an object as a JSON string. """
+
+    return json.dumps(data, default = default, sort_keys = sort_keys, **kwargs)
+
+def dump_path(
+        data: typing.Any,
+        path: str,
+        default: typing.Union[typing.Callable, None] = _custom_handle,
+        sort_keys: bool = True,
+        encoding: str = edq.util.dirent.DEFAULT_ENCODING,
+        **kwargs) -> None:
+    """ Dump an object as a JSON file. """
+
+    with open(path, 'w', encoding = encoding) as file:
+        dump(data, file, default = default, sort_keys = sort_keys, **kwargs)

edq/util/json_test.py

@@ -0,0 +1,228 @@
+import os
+import typing
+
+import edq.testing.unittest
+import edq.util.dirent
+import edq.util.json
+import edq.util.reflection
+
+class TestJSON(edq.testing.unittest.BaseTest):
+    """ Test JSON utils. """
+
+    def test_loading_dumping_base(self):
+        """
+        Test the family of JSON loading and dumping functions.
+        """
+
+        # [(string, dict, strict?, error_substring), ...]
+        test_cases = [
+            # Base
+            (
+                '{"a": 1}',
+                {"a": 1},
+                False,
+                None,
+            ),
+
+            # Trivial - Strict
+            (
+                '{"a": 1}',
+                {"a": 1},
+                True,
+                None,
+            ),
+
+            # JSON5
+            (
+                '{"a": 1,}',
+                {"a": 1},
+                False,
+                None,
+            ),
+
+            # JSON5 - Strict
+            (
+                '{"a": 1,}',
+                {"a": 1},
+                True,
+                'JSONDecodeError',
+            ),
+        ]
+
+        # [(function, name), ...]
+        test_methods = [
+            (self._subtest_loads_dumps, 'subtest_loads_dumps'),
+            (self._subtest_load_dump, 'subtest_load_dump'),
+            (self._subtest_load_dump_path, 'subtest_load_dump_path'),
+        ]
+
+        for (test_method, test_method_name) in test_methods:
+            for (i, test_case) in enumerate(test_cases):
+                (text_content, dict_content, strict, error_substring) = test_case
+
+                with self.subTest(msg = f"Subtest {test_method_name}, Case {i} ('{text_content}'):"):
+                    try:
+                        test_method(text_content, dict_content, strict)
+                    except AssertionError:
+                        # The subttest failed an assertion.
+                        raise
+                    except Exception as ex:
+                        error_string = self.format_error_string(ex)
+                        if (error_substring is None):
+                            self.fail(f"Unexpected error: '{error_string}'.")
+
+                        self.assertIn(error_substring, error_string, 'Error is not as expected.')
+
+                        continue
+
+                    if (error_substring is not None):
+                        self.fail(f"Did not get expected error: '{error_substring}'.")
+
+    def _subtest_loads_dumps(self, text_content, dict_content, strict):
+        actual_dict = edq.util.json.loads(text_content, strict = strict)
+        actual_text = edq.util.json.dumps(dict_content)
+        double_conversion_text = edq.util.json.dumps(actual_dict)
+
+        self.assertDictEqual(dict_content, actual_dict)
+        self.assertEqual(actual_text, double_conversion_text)
+
+    def _subtest_load_dump(self, text_content, dict_content, strict):
+        temp_dir = edq.util.dirent.get_temp_dir(prefix = 'edq_test_json_')
+
+        path_text = os.path.join(temp_dir, 'test-text.json')
+        path_dict = os.path.join(temp_dir, 'test-dict.json')
+
+        edq.util.dirent.write_file(path_text, text_content)
+
+        with open(path_text, 'r', encoding = edq.util.dirent.DEFAULT_ENCODING) as file:
+            text_load = edq.util.json.load(file, strict = strict)
+
+        with open(path_dict, 'w', encoding = edq.util.dirent.DEFAULT_ENCODING) as file:
+            edq.util.json.dump(dict_content, file)
+
+        with open(path_dict, 'r', encoding = edq.util.dirent.DEFAULT_ENCODING) as file:
+            dict_load = edq.util.json.load(file, strict = strict)
+
+        self.assertDictEqual(dict_content, text_load)
+        self.assertDictEqual(dict_load, text_load)
+
+    def _subtest_load_dump_path(self, text_content, dict_content, strict):
+        temp_dir = edq.util.dirent.get_temp_dir(prefix = 'edq_test_json_path_')
+
+        path_text = os.path.join(temp_dir, 'test-text.json')
+        path_dict = os.path.join(temp_dir, 'test-dict.json')
+
+        edq.util.dirent.write_file(path_text, text_content)
+        text_load = edq.util.json.load_path(path_text, strict = strict)
+
+        edq.util.json.dump_path(dict_content, path_dict)
+        dict_load = edq.util.json.load_path(path_dict, strict = strict)
+
+        self.assertDictEqual(dict_content, text_load)
+        self.assertDictEqual(dict_load, text_load)
+
+    def test_object_base(self):
+        """
+        Test loading and dumping JSON objects
+        """
+
+        # [(string, object, error_substring), ...]
+        test_cases = [
+            # Base
+            (
+                '{"a": 1, "b": "b"}',
+                _TestConverter(1, "b"),
+                None,
+            ),
+
+            # Missing Key
+            (
+                '{"a": 1}',
+                _TestConverter(1, None),
+                None,
+            ),
+
+            # Empty
+            (
+                '{}',
+                _TestConverter(None, None),
+                None,
+            ),
+
+            # Extra Key
+            (
+                '{"a": 1, "b": "b", "c": 0}',
+                _TestConverter(1, "b"),
+                None,
+            ),
+
+            # List
+            (
+                '[{"a": 1, "b": "b"}]',
+                _TestConverter(1, "b"),
+                'not a dict',
+            ),
+        ]
+
+        # [(function, name), ...]
+        test_methods = [
+            (self._subtest_loads_object, 'subtest_loads_object'),
+            (self._subtest_load_object_path, 'subtest_load_object_path'),
+        ]
+
+        for (test_method, test_method_name) in test_methods:
+            for (i, test_case) in enumerate(test_cases):
+                (text_content, object_content, error_substring) = test_case
+
+                with self.subTest(msg = f"Subtest {test_method_name}, Case {i} ('{text_content}'):"):
+                    try:
+                        test_method(text_content, object_content)
+                    except AssertionError:
+                        # The subttest failed an assertion.
+                        raise
+                    except Exception as ex:
+                        error_string = self.format_error_string(ex)
+                        if (error_substring is None):
+                            self.fail(f"Unexpected error: '{error_string}'.")
+
+                        self.assertIn(error_substring, error_string, 'Error is not as expected.')
+
+                        continue
+
+                    if (error_substring is not None):
+                        self.fail(f"Did not get expected error: '{error_substring}'.")
+
+    def _subtest_loads_object(self, text_content, object_content):
+        actual_object = edq.util.json.loads_object(text_content, _TestConverter)
+        actual_text = edq.util.json.dumps(object_content)
+        double_conversion_text = edq.util.json.dumps(actual_object)
+
+        self.assertEqual(object_content, actual_object)
+        self.assertEqual(actual_text, double_conversion_text)
+
+    def _subtest_load_object_path(self, text_content, object_content):
+        temp_dir = edq.util.dirent.get_temp_dir(prefix = 'edq_test_json_object_path_')
+
+        path_text = os.path.join(temp_dir, 'test-text.json')
+        path_object = os.path.join(temp_dir, 'test-object.json')
+
+        edq.util.dirent.write_file(path_text, text_content)
+        text_load = edq.util.json.load_object_path(path_text, _TestConverter)
+
+        edq.util.json.dump_path(object_content, path_object)
+        object_load = edq.util.json.load_object_path(path_object, _TestConverter)
+
+        self.assertEqual(object_content, text_load)
+        self.assertEqual(object_load, text_load)
+
+class _TestConverter(edq.util.json.DictConverter):
+    def __init__(self, a: typing.Union[int, None] = None, b: typing.Union[str, None] = None, **kwargs) -> None:
+        self.a: typing.Union[int, None] = a
+        self.b: typing.Union[str, None] = b
+
+    def to_dict(self) -> typing.Dict[str, typing.Any]:
+        return vars(self)
+
+    @classmethod
+    def from_dict(cls, data: typing.Dict[str, typing.Any]) -> typing.Any:
+        return _TestConverter(**data)

edq/util/pyimport.py

@@ -0,0 +1,73 @@
+import importlib
+import importlib.util
+import os
+import typing
+import uuid
+
+import edq.util.dirent
+
+_import_cache: typing.Dict[str, typing.Any] = {}
+""" A cache to help avoid importing a module multiple times. """
+
+def import_path(raw_path: str, cache: bool = True, module_name: typing.Union[str, None] = None) -> typing.Any:
+    """
+    Import a module from a file.
+    If cache is false, then the module will not be fetched or stored in this module's cache.
+    """
+
+    path = os.path.abspath(raw_path)
+    cache_key = f"PATH::{path}"
+
+    # Check the cache before importing.
+    if (cache):
+        module = _import_cache.get(cache_key, None)
+        if (module is not None):
+            return module
+
+    if (not edq.util.dirent.exists(path)):
+        raise ValueError(f"Module path does not exist: '{raw_path}'.")
+
+    if (not os.path.isfile(path)):
+        raise ValueError(f"Module path is not a file: '{raw_path}'.")
+
+    if (module_name is None):
+        module_name = str(uuid.uuid4()).replace('-', '')
+
+    spec = importlib.util.spec_from_file_location(module_name, path)
+    if ((spec is None) or (spec.loader is None)):
+        raise ValueError(f"Failed to load module specification for path: '{raw_path}'.")
+
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    # Store the module in the cache.
+    if (cache):
+        _import_cache[cache_key] = module
+
+    return module
+
+def import_name(module_name: str, cache: bool = True) -> typing.Any:
+    """
+    Import a module from a name.
+    The module must already be in the system's path (sys.path).
+    If cache is false, then the module will not be fetched or stored in this module's cache.
+    """
+
+    cache_key = f"NAME::{module_name}"
+
+    # Check the cache before importing.
+    if (cache):
+        module = _import_cache.get(cache_key, None)
+        if (module is not None):
+            return module
+
+    try:
+        module = importlib.import_module(module_name)
+    except ImportError as ex:
+        raise ValueError(f"Unable to locate module '{module_name}'.") from ex
+
+    # Store the module in the cache.
+    if (cache):
+        _import_cache[cache_key] = module
+
+    return module

edq/util/pyimport_test.py

@@ -0,0 +1,83 @@
+import os
+
+import edq.testing.unittest
+import edq.util.pyimport
+
+THIS_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+PACKAGE_ROOT_DIR = os.path.join(THIS_DIR, '..')
+
+class TestPyImport(edq.testing.unittest.BaseTest):
+    """ Test Python importing operations. """
+
+    def test_import_path_base(self):
+        """ Test importing a module from a path. """
+
+        # [(relative path, error substring), ...]
+        # All paths are relative to the package root.
+        test_cases = [
+            # Standard Module
+            (os.path.join('util', 'pyimport.py'), None),
+
+            # Errors
+            ('ZZZ', 'Module path does not exist'),
+            ('util', 'Module path is not a file'),
+        ]
+
+        for (i, test_case) in enumerate(test_cases):
+            (relpath, error_substring) = test_case
+
+            with self.subTest(msg = f"Case {i} ('{relpath}'):"):
+                path = os.path.join(PACKAGE_ROOT_DIR, relpath)
+
+                try:
+                    module = edq.util.pyimport.import_path(path)
+                except Exception as ex:
+                    error_string = self.format_error_string(ex)
+                    if (error_substring is None):
+                        self.fail(f"Unexpected error: '{error_string}'.")
+
+                    self.assertIn(error_substring, error_string, 'Error is not as expected.')
+
+                    continue
+
+                if (error_substring is not None):
+                    self.fail(f"Did not get expected error: '{error_substring}'.")
+
+                self.assertIsNotNone(module)
+
+    def test_import_name_base(self):
+        """ Test importing a module from a name. """
+
+        # [(name, error substring), ...]
+        test_cases = [
+            # Standard Module
+            ('edq.util.pyimport', None),
+
+            # Package (__init__.py)
+            ('edq.util', None),
+
+            # Errors
+            ('', 'Empty module name'),
+            ('edq.util.ZZZ', 'Unable to locate module'),
+            ('edq.util.pyimport.ZZZ', 'Unable to locate module'),
+        ]
+
+        for (i, test_case) in enumerate(test_cases):
+            (name, error_substring) = test_case
+
+            with self.subTest(msg = f"Case {i} ('{name}'):"):
+                try:
+                    module = edq.util.pyimport.import_name(name)
+                except Exception as ex:
+                    error_string = self.format_error_string(ex)
+                    if (error_substring is None):
+                        self.fail(f"Unexpected error: '{error_string}'.")
+
+                    self.assertIn(error_substring, error_string, 'Error is not as expected.')
+
+                    continue
+
+                if (error_substring is not None):
+                    self.fail(f"Did not get expected error: '{error_substring}'.")
+
+                self.assertIsNotNone(module)

edq/util/reflection.py

@@ -0,0 +1,32 @@
+import typing
+
+def get_qualified_name(target: typing.Union[type, object, str]) -> str:
+    """
+    Try to get a qualified name for a type (or for the type of an object).
+    Names will not always come out clean.
+    """
+
+    # Get the type for this target.
+    if (isinstance(target, type)):
+        target_class = target
+    elif (callable(target)):
+        target_class = typing.cast(type, target)
+    else:
+        target_class = type(target)
+
+    # Check for various name components.
+    parts = []
+
+    if (hasattr(target_class, '__module__')):
+        parts.append(str(getattr(target_class, '__module__')))
+
+    if (hasattr(target_class, '__qualname__')):
+        parts.append(str(getattr(target_class, '__qualname__')))
+    elif (hasattr(target_class, '__name__')):
+        parts.append(str(getattr(target_class, '__name__')))
+
+    # Fall back to just the string reprsentation.
+    if (len(parts) == 0):
+        return str(target_class)
+
+    return '.'.join(parts)

edq/util/time.py

@@ -0,0 +1,75 @@
+import datetime
+import time
+
+PRETTY_SHORT_FORMAT: str = '%Y-%m-%d %H:%M'
+
+class Duration(int):
+    """
+    A Duration represents some length in time in milliseconds.
+    """
+
+    def to_secs(self) -> float:
+        """ Convert the duration to float seconds. """
+
+        return self / 1000.0
+
+    def to_msecs(self) -> int:
+        """ Convert the duration to integer milliseconds. """
+
+        return self
+
+class Timestamp(int):
+    """
+    A Timestamp represent a moment in time (sometimes called "datetimes").
+    Timestamps are internally represented by the number of milliseconds since the
+    (Unix Epoch)[https://en.wikipedia.org/wiki/Unix_time].
+    This is sometimes referred to as "Unix Time".
+    Since Unix Time is in UTC, timestamps do not need to carry timestamp information with them.
+
+    Note that timestamps are just integers with some decoration,
+    so they respond to all normal int functionality.
+    """
+
+    def sub(self, other: 'Timestamp') -> Duration:
+        """ Return a new duration that is the difference of this and the given duration. """
+
+        return Duration(self - other)
+
+    def to_pytime(self, timezone: datetime.timezone = datetime.timezone.utc) -> datetime.datetime:
+        """ Convert this timestamp to a Python datetime in the given timezone (UTC by default). """
+
+        return datetime.datetime.fromtimestamp(self / 1000, timezone)
+
+    def to_local_pytime(self) -> datetime.datetime:
+        """ Convert this timestamp to a Python datetime in the system timezone. """
+
+        local_timezone = datetime.datetime.now().astimezone().tzinfo
+        if ((local_timezone is None) or (not isinstance(local_timezone, datetime.timezone))):
+            raise ValueError("Could not discover local timezone.")
+
+        return self.to_pytime(timezone = local_timezone)
+
+    def pretty(self, short: bool = False, timezone: datetime.timezone = datetime.timezone.utc) -> str:
+        """
+        Get a "pretty" string representation of this timestamp.
+        There is no guarantee that this representation can be parsed back to its original form.
+        """
+
+        pytime = self.to_pytime(timezone = timezone)
+
+        if (short):
+            return pytime.strftime(PRETTY_SHORT_FORMAT)
+
+        return pytime.isoformat(timespec = 'milliseconds')
+
+    @staticmethod
+    def from_pytime(pytime: datetime.datetime) -> 'Timestamp':
+        """ Convert a Python datetime to a timestamp. """
+
+        return Timestamp(int(pytime.timestamp() * 1000))
+
+    @staticmethod
+    def now() -> 'Timestamp':
+        """ Get a Timestamp that represents the current moment. """
+
+        return Timestamp(time.time() * 1000)