vortexpy 4.0.23__py3-none-any.whl

vortex/data_loader/TupleDataLoaderDelegate.py

@@ -0,0 +1,90 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : https://www.synerty.com
+ * Support : support@synerty.com
+"""
+import abc
+from abc import abstractmethod
+from typing import Union, Optional
+
+from twisted.internet.defer import Deferred
+
+from vortex.TupleSelector import TupleSelector
+from vortex.data_loader.TupleDataLoaderTupleABC import TupleDataLoaderTupleABC
+
+
+class TupleDataLoaderDelegateABC(abc.ABC):
+    """Tuple Data Loader Delegate ABC
+
+    ABC for delegate classes to handle the reading and writing for each forms'
+    data. Handlers must be register in
+    (`Tuple`, `TupleDataLoaderDelegateABC`) pairs with the `TupleDataLoader`
+
+    """
+
+    def start(self):
+        pass
+
+    def shutdown(self):
+        pass
+
+    @abstractmethod
+    def createData(
+        self, data: TupleDataLoaderTupleABC
+    ) -> Union[Deferred, TupleSelector]:
+        """Load Data
+
+        The `createData` method is called when data does not exist and needs
+        to be created.
+
+        :param data: The new object to create
+        :param selector: The current tuple selector for the data.
+        :return: The `TupleSelector` or a `Deferred` which eventually
+            results in a `TupleSelector`. The tuple selector must select the new
+            data.
+        """
+        pass
+
+    @abstractmethod
+    def loadData(
+        self, selector: TupleSelector
+    ) -> Union[Deferred, TupleDataLoaderTupleABC]:
+        """Load Data
+
+        The `loadData` method is called when data needs to be loaded from a
+        storage medium. The query is represented by a `TupleSelector` value
+        `selector`.
+
+        :param selector: The `TupleSelector` for which to load data
+        :return: The `Tuple` or a `Deferred` which eventually results in a `Tuple`
+        """
+        pass
+
+    @abstractmethod
+    def storeData(
+        self, data: TupleDataLoaderTupleABC, tupleSelector: TupleSelector
+    ) -> Optional[TupleSelector]:
+        """Store Data
+
+        Store the data into the storage medium. The `data` is the `Tuple` which
+        needs to be written
+
+        :param data: Tuple to write and store
+        :param selector: Previous selector used to get the data
+        :return: TupleSelector: An optional new tuple selector if it has changed
+        """
+        pass
+
+    @abstractmethod
+    def deleteData(self, selector: TupleSelector) -> Deferred:
+        """Delete Data
+
+        Delete the data referenced by the `selector`
+
+        :param selector: `TupleSelector` for data to be deleted
+        :return: Deferred
+        """
+        pass

vortex/data_loader/TupleDataLoaderTupleABC.py

@@ -0,0 +1,10 @@
+import logging
+from abc import ABCMeta
+
+from vortex.Tuple import Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class TupleDataLoaderTupleABC(Tuple, metaclass=ABCMeta):
+    pass

vortex/data_loader/TupleDataLoaderTuples.py

@@ -0,0 +1,53 @@
+from typing import Optional
+
+from vortex.Tuple import addTupleType, TupleField, Tuple
+from vortex.TupleAction import TupleActionABC
+from vortex.TupleSelector import TupleSelector
+from vortex.data_loader.TupleDataLoaderTupleABC import TupleDataLoaderTupleABC
+
+
+@addTupleType
+class _DataLoaderTupleAction(TupleActionABC):
+    __tupleType__ = "vortex._DataLoaderTupleAction"
+
+    CREATE_ACTION = "create"
+    STORE_ACTION = "store"
+    LOAD_ACTION = "load"
+    DELETE_ACTION = "delete"
+
+    userUuid: str = TupleField()
+    delegateUuid: str = TupleField()
+    dataTuple: TupleDataLoaderTupleABC = TupleField()
+    action: str = TupleField()
+    tupleDataSelector: TupleSelector = TupleField()
+
+
+@addTupleType
+class _DataLoaderTupleActionResponseTuple(Tuple):
+    __tupleType__ = "vortex._DataLoaderTupleActionResponseTuple"
+
+    tupleData: Optional[TupleDataLoaderTupleABC] = TupleField()
+    tupleDataSelector: Optional[TupleSelector] = TupleField()
+
+
+@addTupleType
+class _LockDataTupleAction(TupleActionABC):
+    __tupleType__ = "vortex._LockDataTupleAction"
+
+    lock: bool = TupleField()
+    userUuid: str = TupleField()
+    delegateUuid: str = TupleField()
+    tupleDataSelector: TupleSelector = TupleField()
+    liveUpdateDataTuple: TupleDataLoaderTupleABC = TupleField()
+
+
+@addTupleType
+class _DataLockStatusTuple(Tuple):
+    __tupleType__ = "vortex._DataLockStatusTuple"
+
+    locked: bool = TupleField(False)
+    deleted: bool = TupleField(False)
+    lockedByUserUuid: Optional[str] = TupleField()
+    lockAutoExpireDate: Optional[str] = TupleField()
+    liveUpdateDataFromDelegateUuid: Optional[str] = TupleField()
+    liveUpdateDataTuple: Optional[TupleDataLoaderTupleABC] = TupleField()

vortex/event_bus/TupleChangeEventABC.py

@@ -0,0 +1,15 @@
+import json
+from abc import ABCMeta
+
+
+class TupleChangeEventABC(metaclass=ABCMeta):
+    __eventType__ = None
+
+    @classmethod
+    def eventType(cls) -> str:
+        return cls.__eventType__
+
+    def __repr__(self):
+        return json.dumps(
+            {"class": self.__class__.__name__, "fields": self.__dict__}
+        )

vortex/event_bus/TupleChangeEventBus.py

@@ -0,0 +1,192 @@
+import logging
+from _weakrefset import WeakSet
+from collections import defaultdict
+from datetime import datetime
+from typing import Optional, Type, Union
+
+import pytz
+from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks
+
+from vortex.TupleSelector import TupleSelector
+from vortex.event_bus.TupleChangeEventABC import TupleChangeEventABC
+from vortex.event_bus.TupleChangeEventBusObserverABC import (
+    TupleChangeEventBusObserverABC,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class TupleChangeEventBus:
+    def __init__(self):
+        self._weakObserversForAllTupleChangeEventABCNames: WeakSet[
+            TupleChangeEventBusObserverABC
+        ] = WeakSet([])
+
+        self._weakObserversBySelectorName: dict[
+            str, WeakSet[TupleChangeEventBusObserverABC]
+        ] = defaultdict(lambda: WeakSet([]))
+
+    def notifyMany(self, events: list[TupleChangeEventABC]) -> None:
+        for event in events:
+            self.notify(event)
+
+    def notify(self, event: TupleChangeEventABC) -> None:
+        """
+        Notify
+
+        Notify the bus of a change to a tuple.
+
+        This bus does not automatically call
+        TupleDataObservableHandler.notifyOfTupleUpdate(...)
+        that must be done by classes implementing
+        TupleChangeEventBusObserverABC
+
+        IMPORTANT: You can use this bus for whatever you like,
+        however, see best practices.
+
+        BEST PRACTICE: Only notify the bus of precisely
+        the data that has changed, as close to the code that changes it as
+         possible.
+
+        Do not notify the bus of tuple selectors that need updating.
+
+        Classes implementing TupleChangeEventBusObserverABC
+        should observe changed data and themselves
+        decide what tuple selectors they will
+        notify their observers for.
+
+        For example. If you have two tuple selectors:
+
+        1) TupleChangeEventABC(MyTuple, {id:1})
+        2) TupleChangeEventABC(ListOfMyTuples)
+
+        TupleChangeEventBus.notify should be called for MyTuple
+        just after changes are committed.
+
+        TupleChangeEventBus.notify SHOULD NOT BE called for
+        ListOfMyTuples
+
+        A typical observer's method might look like this:
+
+        ```python
+            def notifyFromBus(self, event: TupleChangeEventABC) -> None:
+                if event.isForTuple(MyTuple):
+                    self.tupleDataObservable.notifyOfTupleUpdate(event)
+                    self.tupleDataObservable.notifyOfTupleUpdate(
+                        TupleChangeEventABC(ListOfMyTuples)
+                    )
+                    return
+
+                ...
+
+        ```
+
+        :param event: The tuple selector that describes the tuple
+        that has been changed. This change might be created, updated, or
+        deleted.
+        :return:
+        """
+        weakSets = [self._weakObserversForAllTupleChangeEventABCNames]
+
+        eventName = event.eventType()
+
+        if eventName in self._weakObserversBySelectorName:
+            weakSets.append(list(self._weakObserversBySelectorName[eventName]))
+
+        for weakSet in weakSets:
+            for item in weakSet:
+                if not item:
+                    continue
+
+                # noinspection PyUnresolvedReferences
+                reactor.callLater(0, self._notify, item, event)
+
+    @inlineCallbacks
+    def _notify(
+        self,
+        observable: TupleChangeEventBusObserverABC,
+        event: TupleChangeEventABC,
+    ):
+        try:
+            startDate = datetime.now(pytz.utc)
+            yield observable.notifyFromBus(event)
+
+            # A blocking call taking more than 100ms is BAD
+            # Otherwise a call taking more than a 1s is just poor performance.
+            secondsTaken = (datetime.now(pytz.utc) - startDate).total_seconds()
+            if 0.1 < secondsTaken:
+                logger.debug(
+                    "notifyFromBus took %s for %s, event %s",
+                    secondsTaken,
+                    observable.__class__.__name__,
+                    event,
+                )
+
+        except Exception as e:
+            logger.error(
+                "Observable %s raised an error"
+                " when being called with event %s",
+                observable.__class__.__name__,
+                event,
+            )
+            logger.exception(e)
+
+    def addObserver(
+        self,
+        observer: TupleChangeEventBusObserverABC,
+        eventNameOrClass: Optional[
+            Union[
+                str,
+                Type[TupleChangeEventABC],
+                TupleChangeEventABC,
+            ]
+        ] = None,
+    ) -> None:
+        assert isinstance(
+            observer, TupleChangeEventBusObserverABC
+        ), "observer must be a TupleChangeEventBusObserverABC"
+
+        if eventNameOrClass is None:
+            self._weakObserversForAllTupleChangeEventABCNames.add(observer)
+            return
+
+        eventName = self._nameFromEvent(eventNameOrClass)
+
+        self._weakObserversBySelectorName[eventName].add(observer)
+
+    # noinspection PyMethodMayBeStatic
+    def _nameFromEvent(
+        self,
+        nameOrTuple: Optional[
+            Union[str, Type[TupleChangeEventABC], TupleChangeEventABC]
+        ],
+    ) -> Optional[str]:
+        # When the TupleSelector is reconstructed / deserialize,
+        # it's constructed with
+        # no arguments
+        if nameOrTuple is None:
+            return None
+
+        if isinstance(nameOrTuple, str):
+            name = nameOrTuple
+
+        elif isinstance(nameOrTuple, TupleChangeEventABC):
+            name = nameOrTuple.eventType()
+
+        elif issubclass(nameOrTuple, TupleChangeEventABC):
+            name = nameOrTuple.eventType()
+
+        else:
+            raise Exception(
+                "Argument should be of type "
+                "Union[str, Type[TupleChangeEventABC], TupleChangeEventABC]"
+            )
+
+        if name == TupleSelector.tupleName():
+            raise Exception(
+                "We've found the name of TupleSelector.tupleType(),"
+                "which means we've been passed a TupleSelector."
+            )
+
+        return name

vortex/event_bus/TupleChangeEventBusObserverABC.py

@@ -0,0 +1,11 @@
+from abc import ABCMeta, abstractmethod
+
+from vortex.event_bus.TupleChangeEventABC import TupleChangeEventABC
+
+
+class TupleChangeEventBusObserverABC(metaclass=ABCMeta):
+    @abstractmethod
+    def notifyFromBus(self, event: TupleChangeEventABC) -> None:
+        raise NotImplementedError(
+            "TupleChangeEventBusObserverABC.notifyFromBus"
+        )

vortex/handler/TupleActionProcessor.py

@@ -0,0 +1,270 @@
+import logging
+from abc import ABCMeta
+from abc import abstractmethod
+from inspect import signature
+from typing import Dict
+from typing import Optional
+from typing import Union
+
+from twisted.internet.defer import Deferred
+from twisted.internet.defer import fail
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet.defer import succeed
+from twisted.python import failure
+from twisted.python.failure import Failure
+
+from vortex.DeferUtil import vortexLogFailure
+from vortex.Payload import Payload
+from vortex.PayloadEndpoint import PayloadEndpoint
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.TupleAction import TupleActionABC
+from vortex.TupleActionVortex import TupleActionVortex
+from vortex.VortexABC import SendVortexMsgResponseCallable
+from vortex.VortexFactory import VortexFactory
+from vortex.VortexUtil import limitConcurrency
+
+logger = logging.getLogger(__name__)
+
+
+class TupleActionProcessorDelegateABC(metaclass=ABCMeta):
+    @abstractmethod
+    def processTupleAction(
+        self, tupleAction: TupleActionABC, **kwargs
+    ) -> Deferred:
+        """Process Tuple Action
+
+        The method generates the vortexMsg for the vortex to send.
+
+        :param tupleAction: The C{TupleAction} to process.
+
+        """
+
+
+class TupleActionProcessor:
+    def __init__(
+        self,
+        tupleActionProcessorName: str,
+        additionalFilt: Optional[Dict] = None,
+        defaultDelegate: Optional[TupleActionProcessorDelegateABC] = None,
+        acceptOnlyFromVortex: Optional[Union[str, tuple]] = None,
+        usedForProxy__=False,
+    ) -> None:
+        """Constructor
+
+        :param tupleActionProcessorName: The name of this observable
+
+        :param additionalFilt: Any additional filter keys that are required
+
+        :param defaultDelegate: The default delegate to send all actions to
+
+        :param acceptOnlyFromVortex: Accept requests only from this vortex,
+            The vortex can be str or tuple of str, or None to accept from any.
+        """
+
+        self._tupleActionProcessorName = tupleActionProcessorName
+        self._defaultDelegate = defaultDelegate
+
+        self._tupleProcessorsByTupleName: Dict[
+            str, TupleActionProcessorDelegateABC
+        ] = {}
+
+        if not usedForProxy__:
+            self._filt = dict(
+                name=tupleActionProcessorName, key="tupleActionProcessorName"
+            )
+
+            if additionalFilt:
+                self._filt.update(additionalFilt)
+
+            self._endpoint = PayloadEndpoint(
+                self._filt,
+                self._processLimited,
+                acceptOnlyFromVortex=acceptOnlyFromVortex,
+            )
+
+    def setDelegate(
+        self, tupleName: str, processor: TupleActionProcessorDelegateABC
+    ):
+        """Add Tuple Action Processor Delegate
+
+        :param tupleName: The tuple name to process actions for.
+        :param processor: The processor to use for processing this tuple name.
+
+        """
+        assert (
+            not tupleName in self._tupleProcessorsByTupleName
+        ), "TupleActionProcessor:%s, Tuple name %s is already registered" % (
+            self._tupleActionProcessorName,
+            tupleName,
+        )
+
+        assert isinstance(processor, TupleActionProcessorDelegateABC), (
+            "TupleActionProcessor:%s, provider must be an"
+            " instance of TupleActionProcessorDelegateABC"
+            % self._tupleActionProcessorName
+        )
+
+        self._tupleProcessorsByTupleName[tupleName] = processor
+
+    @property
+    def delegateCount(self) -> int:
+        return len(self._tupleProcessorsByTupleName)
+
+    def hasDelegate(self, tupleName: str) -> bool:
+        return tupleName in self._tupleProcessorsByTupleName
+
+    def shutdown(self):
+        self._endpoint.shutdown()
+
+    def _processLimited(self, *args, **kwargs):
+        d = self._processLimitedDecorated(*args, **kwargs)
+        if d:
+            d.addErrback(vortexLogFailure, logger, consumeError=True)
+
+    @limitConcurrency(logger, 25)
+    def _processLimitedDecorated(self, *args, **kwargs):
+        return self._process(*args, **kwargs)
+
+    @inlineCallbacks
+    def _process(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        vortexUuid: str,
+        sendResponse: SendVortexMsgResponseCallable,
+        **kwargs,
+    ):
+        """Process the Payload / Tuple Action"""
+        if not VortexFactory.isVortexUuidOnline(vortexUuid):
+            logger.debug(
+                "Vortex %s is no longer online, skipping reply", vortexUuid
+            )
+            return
+
+        # logger.debug(
+        #     "Vortex %s is STILL online, continuing to reply", vortexUuid
+        # )
+
+        payload = yield payloadEnvelope.decodePayloadDefer()
+
+        assert (
+            len(payload.tuples) == 1
+        ), "TupleActionProcessor:%s Expected 1 tuples, received %s" % (
+            self._tupleActionProcessorName,
+            len(payload.tuples),
+        )
+
+        tupleAction = payload.tuples[0]
+
+        yield self._processTupleAction(
+            payloadEnvelope.filt,
+            sendResponse,
+            tupleAction,
+            vortexUuid=vortexUuid,
+            **kwargs,
+        )
+
+    def _processTupleAction(
+        self, payloadEnvelopeFilt, sendResponse, tupleAction, **kwargs
+    ):
+
+        assert isinstance(
+            tupleAction, TupleActionABC
+        ), "TupleActionProcessor:%s Expected TupleAction, received %s" % (
+            self._tupleActionProcessorName,
+            tupleAction.__class__,
+        )
+
+        tupleName = tupleAction.tupleName()
+        processor = self._tupleProcessorsByTupleName.get(tupleName)
+        if processor:
+            delegate = processor.processTupleAction
+
+        elif self._defaultDelegate:
+            delegate = self._defaultDelegate.processTupleAction
+
+        else:
+            raise Exception("No delegate registered for %s" % tupleName)
+
+        sig = signature(delegate)
+        if len(sig.parameters) == 2:
+            tupleActionVortex = TupleActionVortex(**kwargs)
+            d = self._customMaybeDeferred(
+                delegate, tupleAction, tupleActionVortex
+            )
+        else:
+            d = self._customMaybeDeferred(delegate, tupleAction)
+
+        d.addCallback(
+            self._callback, payloadEnvelopeFilt, tupleName, sendResponse
+        )
+        d.addErrback(
+            self._errback, payloadEnvelopeFilt, tupleName, sendResponse
+        )
+        return d
+
+    @inlineCallbacks
+    def _callback(
+        self,
+        result,
+        replyFilt: dict,
+        tupleName: str,
+        sendResponse: SendVortexMsgResponseCallable,
+    ):
+
+        if not isinstance(result, list):
+            result = [result]
+        payload = Payload(filt=replyFilt, tuples=result)
+
+        payloadEnvelope = yield payload.makePayloadEnvelopeDefer()
+        vortexMsg = yield payloadEnvelope.toVortexMsgDefer()
+        try:
+            yield sendResponse(vortexMsg)
+
+        except Exception as e:
+            logger.error(
+                "Failed to send TupleAction response for %s", tupleName
+            )
+            logger.exception(e)
+
+    @inlineCallbacks
+    def _errback(
+        self,
+        result: Failure,
+        replyFilt: dict,
+        tupleName: str,
+        sendResponse: SendVortexMsgResponseCallable,
+    ):
+
+        logger.error(
+            "TupleActionProcessor:%s Failed to process TupleActon",
+            self._tupleActionProcessorName,
+        )
+
+        vortexLogFailure(result, logger)
+        failureMessage = result.getErrorMessage()
+
+        payloadEnvelope = PayloadEnvelope(filt=replyFilt, result=failureMessage)
+        vortexMsg = yield payloadEnvelope.toVortexMsgDefer()
+        try:
+            yield sendResponse(vortexMsg)
+
+        except Exception as e:
+            logger.error(
+                "Failed to send TupleAction response for %s\n%s",
+                tupleName,
+                failureMessage,
+            )
+            logger.exception(e)
+
+    def _customMaybeDeferred(self, f, *args, **kw):
+        try:
+            result = f(*args, **kw)
+        except Exception as e:
+            return fail(failure.Failure(e))
+
+        if isinstance(result, Deferred):
+            return result
+        elif isinstance(result, failure.Failure):
+            return fail(result)
+        else:
+            return succeed(result)