vortexpy 4.2.1__tar.gz → 4.3.0__tar.gz

{vortexpy-4.2.1 → vortexpy-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortexpy
-Version: 4.2.1
+Version: 4.3.0
 Summary: Flying Ion's observable, routable, data serialisation and transport code.
 Author-email: Flying Ion Pty Ltd <contact@flyingion.com>
 License: MIT

{vortexpy-4.2.1 → vortexpy-4.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vortexpy"
-version = "4.2.1"
+version = "4.3.0"
 license = {text = "MIT"}
 authors = [{name = "Flying Ion Pty Ltd", email = "contact@flyingion.com"}]
 description = "Flying Ion's observable, routable, data serialisation and transport code."

{vortexpy-4.2.1 → vortexpy-4.3.0}/vortex/Tuple.py

@@ -383,6 +383,13 @@ class _TupleToPlainJsonMixin:
             self, includeNones, includeFalse
         )
 
+    def tupleToBsonJsonDict(self, includeNones=True, includeFalse=True):
+        from vortex.TupleBson import TupleBsonConverter
+
+        return TupleBsonConverter.tupleToBsonJsonDict(
+            self, includeNones, includeFalse
+        )
+
     def __tupleToJsonDict(
         self, includeNones=True, includeFalse=True, useShortNames=True
     ):
@@ -506,6 +513,31 @@ class _TupleToPlainJsonMixin:
             jsonDict, TupleClass
         )
 
+    @staticmethod
+    def bsonJsonDictToTupleWithValidation(
+        jsonDict: dict, TupleClass: Type["Tuple"]
+    ) -> "Tuple":
+        """BSON document dict to Tuple with validation.
+
+        This method uses the typings on the fields in a tuple to validate the
+        types of data.
+        """
+        from vortex.TupleBson import TupleBsonConverter
+
+        return TupleBsonConverter.bsonJsonDictToTupleWithValidation(
+            jsonDict, TupleClass
+        )
+
+    @staticmethod
+    def bsonJsonDictToTupleWithValidationResult(
+        jsonDict: dict, TupleClass: Type["Tuple"]
+    ) -> "TupleValidationResult":
+        from vortex.TupleBson import TupleBsonConverter
+
+        return TupleBsonConverter.bsonJsonDictToTupleWithValidationResult(
+            jsonDict, TupleClass
+        )
+
 
 class Tuple(Jsonable, _TupleToPlainJsonMixin, _TupleToSqlaJsonMixin):
     """Tuple

vortexpy-4.3.0/vortex/TupleBson.py

@@ -0,0 +1,182 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+#
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+
+"""
+BSON document serialisation / deserialisation for vortex.Tuple.
+
+Datetime fields use UTC epoch milliseconds on write and read, matching MongoDB
+BSON UTC datetime. Native ``datetime`` values are also accepted on read.
+Naive datetimes are treated as UTC at this boundary.
+
+Implementation module; public interface remains on vortex.Tuple.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any, Type, get_args
+
+from .SerialiseUtil import WKBElement
+from .SerialiseUtil import convertFromWkbElement
+from .TupleDocumentConverter import TupleDocumentConverterBase
+from .TupleDocumentConverter import TupleValidationError
+from .TupleDocumentConverter import TupleValidationResult
+
+if TYPE_CHECKING:
+    from .Tuple import Tuple
+
+_MIN_EPOCH_MILLISECONDS = 1_000_000_000_000
+
+
+class TupleBsonConverter(TupleDocumentConverterBase):
+    @classmethod
+    def _ensureAwareDatetime(cls, value: datetime) -> datetime:
+        """
+        Return a timezone-aware datetime for BSON serialisation.
+
+        Naive datetimes are treated as UTC.
+        """
+        if value.tzinfo is None:
+            return value.replace(tzinfo=UTC)
+        return value
+
+    @classmethod
+    def _datetimeToBson(cls, value: datetime) -> int:
+        """Serialise a datetime to UTC epoch milliseconds."""
+        aware = cls._ensureAwareDatetime(value)
+        return int(aware.timestamp() * 1000)
+
+    @classmethod
+    def _datetimeFromBson(cls, value: object) -> datetime:
+        """
+        Deserialise a BSON datetime field to UTC.
+
+        Accepts a ``datetime`` or epoch-millisecond ``int`` (not ``bool``).
+        Epoch-second integers are rejected.
+        """
+        if isinstance(value, datetime):
+            return cls._ensureAwareDatetime(value)
+        if isinstance(value, int) and not isinstance(value, bool):
+            if value < _MIN_EPOCH_MILLISECONDS:
+                raise TypeError(
+                    "Expected epoch milliseconds since Unix epoch in UTC,"
+                    f" got {value}"
+                )
+            return datetime.fromtimestamp(value / 1000, tz=UTC)
+        raise TypeError(
+            "Expected BSON datetime or epoch milliseconds,"
+            f" got {type(value).__name__}"
+        )
+
+    @classmethod
+    def _utilCoercePlainDictItem(cls, item: Any) -> Any:
+        if isinstance(item, int) and not isinstance(item, bool):
+            try:
+                return cls._datetimeFromBson(item)
+            except TypeError:
+                return item
+        return item
+
+    @classmethod
+    def _utilCoerceFieldDatetimeValue(
+        cls,
+        field_path: str,
+        value: Any,
+        typeHint: Any,
+    ) -> tuple[Any, TupleValidationError | None]:
+        if typeHint is datetime or datetime in get_args(typeHint):
+            if isinstance(value, (datetime, int)) and not isinstance(value, bool):
+                try:
+                    return cls._datetimeFromBson(value), None
+                except TypeError as exc:
+                    return value, cls._makeError(field_path, str(exc), TypeError)
+        return value, None
+
+    @classmethod
+    def _jsonDictToTupleWithValidationResult(
+        cls, jsonDict: dict[str, Any], TupleClass: Type[Tuple]
+    ) -> TupleValidationResult:
+        return cls.bsonJsonDictToTupleWithValidationResult(jsonDict, TupleClass)
+
+    @classmethod
+    def tupleToBsonJsonDict(
+        cls,
+        tuple_: Tuple,
+        includeNones: bool = True,
+        includeFalse: bool = True,
+    ) -> dict[str, Any]:
+        """
+        Serialise a tuple to a BSON-shaped document dict.
+
+        Datetime fields become UTC epoch-millisecond integers.
+        """
+        if not tuple_.__shortFieldNamesMap__:
+            raise Exception(
+                "Tuple %s has no shortFieldNames defined" % tuple_.tupleType()
+            )
+
+        json: dict[str, Any] = {}
+
+        def convert(value: Any) -> Any:
+            if isinstance(value, list):
+                return [convert(item) for item in value]
+
+            if isinstance(value, dict):
+                return {
+                    convert(key): convert(item) for key, item in value.items()
+                }
+
+            from .Tuple import Tuple
+
+            if isinstance(value, Tuple):
+                return cls.tupleToBsonJsonDict(
+                    value, includeNones, includeFalse
+                )
+
+            if isinstance(value, WKBElement):
+                return convertFromWkbElement(value)
+
+            from .Tuple import TupleField
+
+            if isinstance(value, TupleField):
+                return None
+
+            if isinstance(value, datetime):
+                return cls._datetimeToBson(value)
+
+            return value
+
+        for shortName, normalName in tuple_.__shortFieldNamesMap__.items():
+            value = convert(getattr(tuple_, normalName))
+            if value is None and not includeNones:
+                continue
+            if value is False and not includeFalse:
+                continue
+            json[normalName] = value
+
+        return json
+
+    @classmethod
+    def bsonJsonDictToTupleWithValidationResult(
+        cls, jsonDict: dict[str, Any], TupleClass: Type[Tuple]
+    ) -> TupleValidationResult:
+        """
+        Deserialise a BSON-shaped document dict into a tuple with validation.
+        """
+        return cls.documentJsonDictToTupleWithValidationResult(
+            jsonDict, TupleClass
+        )
+
+    @classmethod
+    def bsonJsonDictToTupleWithValidation(
+        cls, jsonDict: dict[str, Any], TupleClass: Type[Tuple]
+    ) -> Tuple:
+        """
+        Deserialise a BSON-shaped document dict, raising on the first error.
+        """
+        return cls.documentJsonDictToTupleWithValidation(jsonDict, TupleClass)

vortexpy-4.3.0/vortex/TupleBson_test.py

@@ -0,0 +1,353 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+#
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+
+from datetime import UTC, datetime
+from enum import IntEnum, StrEnum
+from typing import Dict, Annotated
+from typing import List
+from typing import Union
+
+from ddt import data, ddt
+from twisted.trial import unittest
+
+from vortex.Tuple import Tuple
+from vortex.Tuple import TupleField
+from vortex.Tuple import TupleHash, IntTupleFieldValidator
+from vortex.Tuple import addTupleType
+from vortex.TupleBson import TupleBsonConverter
+
+
+@addTupleType
+class BsonTestTuple(Tuple):
+    __tupleType__ = "flyingion.vortex.BsonTestTuple"
+    aInt: int = TupleField()
+    aIntWithValidator: Annotated[int, IntTupleFieldValidator(1, 100)] = (
+        TupleField()
+    )
+    aFloat: float = TupleField()
+    aString: str = TupleField()
+    aBoolTrue: bool = TupleField()
+    aBoolFalse: bool = TupleField()
+    aList: List[Union[str, int, bool, float]] = TupleField()
+    aDict: Dict[str, Union[str, float]] = TupleField()
+    aSubTuple: "BsonSubTestTuple" = TupleField()
+    aListOfSubTuples: List["BsonSubTestTuple"] = TupleField()
+    aDictOfSubTuples: Dict[str, "BsonSubTestTuple"] = TupleField()
+
+
+class BsonObjectTypeE(StrEnum):
+    EVENT = "event"
+
+
+class BsonPriorityE(IntEnum):
+    HIGH = 1
+
+
+@addTupleType
+class BsonEnumTestTuple(Tuple):
+    __tupleType__ = "flyingion.vortex.BsonEnumTestTuple"
+    objectType: BsonObjectTypeE = TupleField()
+    optionalObjectType: BsonObjectTypeE | None = TupleField()
+    priority: BsonPriorityE = TupleField()
+    priorities: List[BsonPriorityE] = TupleField()
+
+
+@addTupleType
+class BsonOptionalDatetimeTestTuple(Tuple):
+    __tupleType__ = "flyingion.vortex.BsonOptionalDatetimeTestTuple"
+    requiredDate: datetime = TupleField()
+    optionalDate: datetime | None = TupleField()
+
+
+@addTupleType
+class BsonSubTestTuple(Tuple):
+    __tupleType__ = "flyingion.vortex.BsonSubTestTuple"
+    subInt: int = TupleField()
+
+
+def _makeBsonTestTuple():
+    tuple_ = BsonTestTuple()
+    tuple_.aBoolTrue = True
+    tuple_.aBoolFalse = False
+    tuple_.aFloat = 2.56
+    tuple_.aInt = 1231231
+    tuple_.aIntWithValidator = 50
+    tuple_.aString = "test string from 345345345@$#%#$%#$%#$%#"
+    tuple_.aList = ["y", 3, True, 3.3]
+    tuple_.aDict = {
+        "a": "test str",
+        "1": 1.234,
+    }
+    tuple_.aDictOfSubTuples = {}
+
+    subTuple = BsonSubTestTuple()
+    subTuple.subInt = 2048
+    tuple_.aSubTuple = subTuple
+
+    tuple_.aListOfSubTuples = []
+    for x in range(2):
+        subTuple = BsonSubTestTuple()
+        subTuple.subInt = x
+        tuple_.aListOfSubTuples.append(subTuple)
+
+        subTuple = BsonSubTestTuple()
+        subTuple.subInt = x + 10
+        tuple_.aDictOfSubTuples[str(x)] = subTuple
+
+    return tuple_
+
+
+def _validBsonDict():
+    return _makeBsonTestTuple().tupleToBsonJsonDict()
+
+
+@ddt
+class TupleBsonTest(unittest.TestCase):
+    def _testSerialisation(self, origTuple):
+        bsonDict = origTuple.tupleToBsonJsonDict()
+        deserialisedTuple = origTuple.bsonJsonDictToTupleWithValidation(
+            bsonDict, type(origTuple)
+        )
+        return deserialisedTuple
+
+    @data(-1, 110)
+    def testIntValidatorInvalid(self, int_):
+        origTuple = _makeBsonTestTuple()
+        origTuple.aIntWithValidator = int_
+        self.assertRaises(ValueError, self._testSerialisation, origTuple)
+
+    @data(1, 99)
+    def testIntValidatorValid(self, int_):
+        origTuple = _makeBsonTestTuple()
+        origTuple.aIntWithValidator = int_
+        self.assertEqual(
+            self._testSerialisation(origTuple).aIntWithValidator, int_
+        )
+
+    def testValidatorPass(self):
+        origTuple = _makeBsonTestTuple()
+        deserialisedTuple = self._testSerialisation(origTuple)
+
+        self.assertEqual(TupleHash(origTuple), TupleHash(origTuple))
+        self.assertEqual(
+            TupleHash(deserialisedTuple), TupleHash(deserialisedTuple)
+        )
+        self.assertEqual(
+            TupleHash(origTuple),
+            TupleHash(deserialisedTuple),
+            "Tuple serialise -> deserialise error",
+        )
+
+    def testDatetimeRoundTrip(self):
+        origTuple = BsonOptionalDatetimeTestTuple()
+        origTuple.requiredDate = datetime(
+            2020, 1, 2, 3, 4, 5, 678000, tzinfo=UTC
+        )
+        origTuple.optionalDate = datetime(
+            2021, 2, 3, 4, 5, 6, 789000, tzinfo=UTC
+        )
+
+        bsonDict = origTuple.tupleToBsonJsonDict()
+        self.assertIsInstance(bsonDict["requiredDate"], int)
+        self.assertIsInstance(bsonDict["optionalDate"], int)
+
+        deserialisedTuple = Tuple.bsonJsonDictToTupleWithValidation(
+            bsonDict, BsonOptionalDatetimeTestTuple
+        )
+        self.assertEqual(
+            deserialisedTuple.requiredDate, origTuple.requiredDate
+        )
+        self.assertEqual(
+            deserialisedTuple.optionalDate, origTuple.optionalDate
+        )
+
+    def testDatetimeFromEpochMilliseconds(self):
+        epochMs = 1577934245678
+        deserialisedTuple = Tuple.bsonJsonDictToTupleWithValidation(
+            dict(requiredDate=epochMs, optionalDate=None),
+            BsonOptionalDatetimeTestTuple,
+        )
+        self.assertEqual(
+            deserialisedTuple.requiredDate,
+            datetime(2020, 1, 2, 3, 4, 5, 678000, tzinfo=UTC),
+        )
+        self.assertIsNone(deserialisedTuple.optionalDate)
+
+    def testDatetimeFromEpochSecondsRejected(self):
+        epochSeconds = 1577934245
+        with self.assertRaises(TypeError) as ctx:
+            Tuple.bsonJsonDictToTupleWithValidation(
+                dict(requiredDate=epochSeconds, optionalDate=None),
+                BsonOptionalDatetimeTestTuple,
+            )
+        self.assertIn("epoch milliseconds", str(ctx.exception))
+
+    def testDatetimeFromBoolRejected(self):
+        result = Tuple.bsonJsonDictToTupleWithValidationResult(
+            dict(requiredDate=True, optionalDate=None),
+            BsonOptionalDatetimeTestTuple,
+        )
+        self.assertEqual(len(result.errors), 1)
+        self.assertEqual(result.errors[0].field_path, "requiredDate")
+        self.assertEqual(result.errors[0].exc_type, TypeError)
+
+    def testOptionalDatetimeRoundTrip(self):
+        origTuple = BsonOptionalDatetimeTestTuple()
+        origTuple.requiredDate = datetime(
+            2024, 6, 15, 10, 30, 45, 123000, tzinfo=UTC
+        )
+        origTuple.optionalDate = datetime(
+            2024, 6, 16, 11, 31, 46, 654000, tzinfo=UTC
+        )
+
+        bsonDict = origTuple.tupleToBsonJsonDict()
+        deserialisedTuple = Tuple.bsonJsonDictToTupleWithValidation(
+            bsonDict, BsonOptionalDatetimeTestTuple
+        )
+
+        self.assertEqual(deserialisedTuple.requiredDate, origTuple.requiredDate)
+        self.assertEqual(deserialisedTuple.optionalDate, origTuple.optionalDate)
+
+    def testStrEnumAndIntEnumRoundTrip(self):
+        origTuple = BsonEnumTestTuple()
+        origTuple.objectType = BsonObjectTypeE.EVENT
+        origTuple.optionalObjectType = BsonObjectTypeE.EVENT
+        origTuple.priority = BsonPriorityE.HIGH
+        origTuple.priorities = [BsonPriorityE.HIGH]
+
+        bsonDict = origTuple.tupleToBsonJsonDict()
+        deserialisedTuple = Tuple.bsonJsonDictToTupleWithValidation(
+            bsonDict, BsonEnumTestTuple
+        )
+
+        self.assertEqual(deserialisedTuple.objectType, BsonObjectTypeE.EVENT)
+        self.assertEqual(
+            deserialisedTuple.optionalObjectType, BsonObjectTypeE.EVENT
+        )
+        self.assertEqual(deserialisedTuple.priority, BsonPriorityE.HIGH)
+        self.assertEqual(deserialisedTuple.priorities, [BsonPriorityE.HIGH])
+
+    def testOptionalDatetimeNoneRoundTrip(self):
+        origTuple = BsonOptionalDatetimeTestTuple()
+        origTuple.requiredDate = datetime(
+            2024, 6, 15, 10, 30, 45, 123000, tzinfo=UTC
+        )
+        origTuple.optionalDate = None
+
+        bsonDict = origTuple.tupleToBsonJsonDict(includeNones=False)
+        deserialisedTuple = Tuple.bsonJsonDictToTupleWithValidation(
+            bsonDict, BsonOptionalDatetimeTestTuple
+        )
+
+        self.assertEqual(deserialisedTuple.requiredDate, origTuple.requiredDate)
+        self.assertIsNone(deserialisedTuple.optionalDate)
+
+    def testDatetimeFromBsonDirect(self):
+        aware = datetime(2020, 1, 2, 3, 4, 5, 678000, tzinfo=UTC)
+        self.assertEqual(TupleBsonConverter._datetimeFromBson(1577934245678), aware)
+
+
+class TupleBsonCollectingTest(unittest.TestCase):
+    def testSingleBadScalarCollectsErrorAndPopulatesOtherFields(self):
+        bsonDict = _validBsonDict()
+        bsonDict["aIntWithValidator"] = 110
+
+        result = Tuple.bsonJsonDictToTupleWithValidationResult(
+            bsonDict, BsonTestTuple
+        )
+
+        self.assertEqual(len(result.errors), 1)
+        self.assertEqual(result.errors[0].field_path, "aIntWithValidator")
+        self.assertTrue(
+            result.errors[0].message.startswith("At aIntWithValidator:")
+        )
+        self.assertEqual(result.errors[0].exc_type, ValueError)
+        self.assertEqual(result.tuple_.aInt, bsonDict["aInt"])
+        self.assertEqual(result.tuple_.aString, bsonDict["aString"])
+        self.assertEqual(result.tuple_.aIntWithValidator, None)
+
+    def testMultipleTopLevelErrors(self):
+        bsonDict = _validBsonDict()
+        bsonDict["aInt"] = "not-an-int"
+        bsonDict["aIntWithValidator"] = -1
+
+        result = Tuple.bsonJsonDictToTupleWithValidationResult(
+            bsonDict, BsonTestTuple
+        )
+
+        self.assertEqual(len(result.errors), 2)
+        paths = {err.field_path for err in result.errors}
+        self.assertEqual(paths, {"aInt", "aIntWithValidator"})
+        for err in result.errors:
+            self.assertTrue(err.message.startswith(f"At {err.field_path}:"))
+        self.assertEqual(result.tuple_.aString, bsonDict["aString"])
+        self.assertEqual(result.tuple_.aFloat, bsonDict["aFloat"])
+
+    def testListPartialSkipsBadItem(self):
+        bsonDict = _validBsonDict()
+        bsonDict["aListOfSubTuples"] = [
+            {"subInt": 1},
+            "not-a-tuple",
+        ]
+
+        result = Tuple.bsonJsonDictToTupleWithValidationResult(
+            bsonDict, BsonTestTuple
+        )
+
+        self.assertEqual(len(result.tuple_.aListOfSubTuples), 1)
+        self.assertEqual(result.tuple_.aListOfSubTuples[0].subInt, 1)
+        self.assertEqual(len(result.errors), 1)
+        self.assertEqual(result.errors[0].field_path, "aListOfSubTuples[1]")
+        self.assertIn("aListOfSubTuples[1]", result.errors[0].message)
+
+    def testNestedTuplePartialError(self):
+        bsonDict = _validBsonDict()
+        bsonDict["aSubTuple"] = {"subInt": "bad"}
+
+        result = Tuple.bsonJsonDictToTupleWithValidationResult(
+            bsonDict, BsonTestTuple
+        )
+
+        self.assertEqual(result.tuple_.aInt, bsonDict["aInt"])
+        self.assertEqual(result.tuple_.aString, bsonDict["aString"])
+        self.assertIsNone(result.tuple_.aSubTuple.subInt)
+        self.assertEqual(len(result.errors), 1)
+        self.assertEqual(result.errors[0].field_path, "aSubTuple.subInt")
+        self.assertTrue(
+            result.errors[0].message.startswith("At aSubTuple.subInt:")
+        )
+
+    def testInvalidStrEnumValueCollectsError(self):
+        bsonDict = {
+            "objectType": "not-a-valid-object-type",
+            "optionalObjectType": None,
+            "priority": 1,
+            "priorities": [],
+        }
+
+        result = Tuple.bsonJsonDictToTupleWithValidationResult(
+            bsonDict, BsonEnumTestTuple
+        )
+
+        self.assertEqual(len(result.errors), 1)
+        self.assertEqual(result.errors[0].field_path, "objectType")
+        self.assertEqual(result.errors[0].exc_type, ValueError)
+
+    def testStrictWrapperRaisesFirstCollectingError(self):
+        bsonDict = _validBsonDict()
+        bsonDict["aInt"] = "not-an-int"
+        bsonDict["aIntWithValidator"] = -1
+
+        result = Tuple.bsonJsonDictToTupleWithValidationResult(
+            bsonDict, BsonTestTuple
+        )
+
+        with self.assertRaises(result.errors[0].exc_type) as ctx:
+            Tuple.bsonJsonDictToTupleWithValidation(bsonDict, BsonTestTuple)
+
+        self.assertEqual(str(ctx.exception), result.errors[0].message)