shapeshifter-uftp 1.0.0__tar.gz

shapeshifter-uftp-1.0.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.1
+Name: shapeshifter-uftp
+Version: 1.0.0
+Summary: Allows connections between DSO, AGR and CRO using the Shapeshifter (UFTP) protocol.
+Home-page: UNKNOWN
+License: UNKNOWN
+Platform: UNKNOWN
+Requires-Python: >=3.10
+Provides-Extra: dev
+
+UNKNOWN
+

shapeshifter-uftp-1.0.0/README.rst

@@ -0,0 +1,28 @@
+Shapeshifter library for Python
+===============================
+
+This is a Python implementation of the ShapeShifter UFTP protocol.
+
+Installation
+------------
+
+.. code-block:: python3
+
+    pip install shapeshifter-uftp
+
+
+Overview
+--------
+
+This library implements the full UFTP protocol that you can use for Shapeshifter communications. It implements all three roles: Distribution System Operator (**DSO**), Aggregator (**AGR**) and Common Reference Operator (**CRO**) in both directions (client and service).
+
+Features of this package:
+
+- Building, parsing and validation of the XML messages
+- Signing and verifying of the XML messages using signatures
+- DNS for service discovery and key retrieval
+- Convenient clients for each role-pair
+- Convenient services for each role
+- JSON-serializable dataclasses for easy transport to other systems
+- Fully internal queing system for full-duplex communication with minimal user code required
+

shapeshifter-uftp-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

shapeshifter-uftp-1.0.0/setup.py

@@ -0,0 +1,40 @@
+from setuptools import setup
+
+setup(
+    name="shapeshifter-uftp",
+    version="1.0.0",
+    python_requires=">=3.10",
+    description="Allows connections between DSO, AGR and CRO using the Shapeshifter (UFTP) protocol.",
+    packages=[
+        "shapeshifter_uftp",
+        "shapeshifter_uftp.uftp",
+        "shapeshifter_uftp.client",
+        "shapeshifter_uftp.service",
+    ],
+    install_requires=[
+        "xsdata[lxml]",
+        "pynacl",
+        "dnspython",
+        "fastapi<0.100",
+        "fastapi-xml",
+        "requests",
+        "uvicorn",
+        "termcolor",
+    ],
+    extras_require={
+        "dev": [
+            "xmlschema",
+            "pytest",
+            "pytest-cov",
+            "pylint",
+            "sphinx",
+            "sphinx-rtd-theme",
+        ]
+    },
+    entry_points={
+        "console_scripts": [
+            "shapeshifter-keypair = shapeshifter_uftp.cli:generate_signing_keypair",
+            "shapeshifter-lookup = shapeshifter_uftp.cli:perform_lookup",
+        ]
+    },
+)

shapeshifter-uftp-1.0.0/shapeshifter_uftp/__init__.py

@@ -0,0 +1,145 @@
+from .client import (
+    ShapeshifterAgrCroClient,
+    ShapeshifterAgrDsoClient,
+    ShapeshifterCroAgrClient,
+    ShapeshifterCroDsoClient,
+    ShapeshifterDsoAgrClient,
+    ShapeshifterDsoCroClient,
+)
+from .service import (
+    ShapeshifterAgrService,
+    ShapeshifterDsoService,
+    ShapeshifterCroService,
+)
+from .uftp import (
+    AcceptedRejected,
+    AgrPortfolioQuery,
+    AgrPortfolioQueryResponse,
+    AgrPortfolioQueryResponseCongestionPoint,
+    AgrPortfolioQueryResponseConnection,
+    AgrPortfolioQueryResponseDSOPortfolio,
+    AgrPortfolioQueryResponseDSOView,
+    AgrPortfolioUpdate,
+    AgrPortfolioUpdateConnection,
+    AgrPortfolioUpdateResponse,
+    ContractSettlement,
+    ContractSettlementISP,
+    ContractSettlementPeriod,
+    DPrognosis,
+    DPrognosisISP,
+    DPrognosisResponse,
+    DsoPortfolioQuery,
+    DsoPortfolioQueryCongestionPoint,
+    DsoPortfolioQueryConnection,
+    DsoPortfolioQueryResponse,
+    DsoPortfolioUpdate,
+    DsoPortfolioUpdateCongestionPoint,
+    DsoPortfolioUpdateConnection,
+    DsoPortfolioUpdateResponse,
+    FlexMessage,
+    FlexOffer,
+    FlexOfferOption,
+    FlexOfferOptionISP,
+    FlexOfferResponse,
+    FlexOfferRevocation,
+    FlexOfferRevocationResponse,
+    FlexOrder,
+    FlexOrderISP,
+    FlexOrderResponse,
+    FlexOrderSettlement,
+    FlexOrderSettlementISP,
+    FlexOrderSettlementStatus,
+    FlexOrderStatus,
+    FlexRequest,
+    FlexRequestISP,
+    FlexRequestResponse,
+    FlexReservationUpdate,
+    FlexReservationUpdateISP,
+    FlexReservationUpdateResponse,
+    FlexSettlement,
+    FlexSettlementResponse,
+    Metering,
+    MeteringISP,
+    # MeteringMessage,
+    MeteringProfile,
+    MeteringProfileEnum,
+    MeteringResponse,
+    MeteringUnit,
+    PayloadMessage,
+    PayloadMessageResponse,
+    SignedMessage,
+    TestMessage,
+    TestMessageResponse,
+    UsefRole,
+)
+
+__all__ = [
+    "ShapeshifterAgrCroClient",
+    "ShapeshifterAgrDsoClient",
+    "ShapeshifterCroAgrClient",
+    "ShapeshifterCroDsoClient",
+    "ShapeshifterDsoAgrClient",
+    "ShapeshifterDsoCroClient",
+    "ShapeshifterAgrService",
+    "ShapeshifterDsoService",
+    "ShapeshifterCroService",
+    "AcceptedRejected",
+    "AgrPortfolioQuery",
+    "AgrPortfolioQueryResponse",
+    "AgrPortfolioQueryResponseCongestionPoint",
+    "AgrPortfolioQueryResponseConnection",
+    "AgrPortfolioQueryResponseDSOPortfolio",
+    "AgrPortfolioQueryResponseDSOView",
+    "AgrPortfolioUpdate",
+    "AgrPortfolioUpdateConnection",
+    "AgrPortfolioUpdateResponse",
+    "ContractSettlement",
+    "ContractSettlementISP",
+    "ContractSettlementPeriod",
+    "DPrognosis",
+    "DPrognosisISP",
+    "DPrognosisResponse",
+    "DsoPortfolioQuery",
+    "DsoPortfolioQueryCongestionPoint",
+    "DsoPortfolioQueryConnection",
+    "DsoPortfolioQueryResponse",
+    "DsoPortfolioUpdate",
+    "DsoPortfolioUpdateCongestionPoint",
+    "DsoPortfolioUpdateConnection",
+    "DsoPortfolioUpdateResponse",
+    "FlexMessage",
+    "FlexOffer",
+    "FlexOfferOption",
+    "FlexOfferOptionISP",
+    "FlexOfferResponse",
+    "FlexOfferRevocation",
+    "FlexOfferRevocationResponse",
+    "FlexOrder",
+    "FlexOrderISP",
+    "FlexOrderResponse",
+    "FlexOrderSettlement",
+    "FlexOrderSettlementISP",
+    "FlexOrderSettlementStatus",
+    "FlexOrderStatus",
+    "FlexRequest",
+    "FlexRequestISP",
+    "FlexRequestResponse",
+    "FlexReservationUpdate",
+    "FlexReservationUpdateISP",
+    "FlexReservationUpdateResponse",
+    "FlexSettlement",
+    "FlexSettlementResponse",
+    "Metering",
+    "MeteringISP",
+    # "MeteringMessage",
+    "MeteringProfile",
+    "MeteringProfileEnum",
+    "MeteringResponse",
+    "MeteringUnit",
+    "PayloadMessage",
+    "PayloadMessageResponse",
+    "SignedMessage",
+    "TestMessage",
+    "TestMessageResponse",
+    "UsefRole",
+]

shapeshifter-uftp-1.0.0/shapeshifter_uftp/cli.py

@@ -0,0 +1,59 @@
+"""
+A set of command-line-interface functions that are useful during
+development of Shapeshifter applications.
+"""
+
+from argparse import ArgumentParser
+from base64 import b64encode
+from nacl.bindings import crypto_sign_keypair
+from . import transport
+from .exceptions import ServiceDiscoveryException, AuthenticationTimeoutException
+
+def generate_signing_keypair():
+    """
+    Generate a signing keypair (private and public) and print them as
+    base64-encoded strings. These are the strings that you'd use for
+    signing and verifying messages; you pass these to the signing_key
+    and recipient_signing_key parameters of the Service or Client
+    objects.
+    """
+    public, private = crypto_sign_keypair()
+    print("-" * 66)
+    print("Private key (base64):", b64encode(private).decode())
+    print("Public key (base64): ", b64encode(public).decode())
+    print("-" * 66)
+
+def perform_lookup():
+    """
+    Perform a DNS lookup of a participant's version, endpoint and
+    public key details. These use the well-known DNS names described
+    in the UFTP specification.
+    """
+    parser = ArgumentParser()
+    parser.add_argument("-d", "--domain", required=True, type=str, help="The sender domain for the other party")
+    parser.add_argument("-r", "--role", required=True, type=str, help="The sender role for the other party")
+    args = parser.parse_args()
+
+    print("-" * 65)
+
+    try:
+        version = transport.get_version(args.domain)
+        print(f"Shapeshifer version: {version}")
+    except ServiceDiscoveryException as err:
+        print(err)
+
+    try:
+        endpoint = transport.get_endpoint(args.domain, args.role)
+        print(f"Endpoint URL:        {endpoint}")
+    except ServiceDiscoveryException as err:
+        print(err)
+
+    try:
+        signing_key, decryption_key = transport.get_keys(args.domain, args.role)
+        print(f"Signing key:         {signing_key}")
+        if decryption_key:
+            print(f"Decryption Key:      {decryption_key}")
+    except AuthenticationTimeoutException as err:
+        print(err)
+
+    print("-" * 65)

shapeshifter-uftp-1.0.0/shapeshifter_uftp/client/__init__.py

@@ -0,0 +1,20 @@
+from .agr_cro_client import ShapeshifterAgrCroClient
+from .agr_dso_client import ShapeshifterAgrDsoClient
+from .cro_agr_client import ShapeshifterCroAgrClient
+from .cro_dso_client import ShapeshifterCroDsoClient
+from .dso_agr_client import ShapeshifterDsoAgrClient
+from .dso_cro_client import ShapeshifterDsoCroClient
+
+__all__ = [
+    "ShapeshifterAgrCroClient",
+    "ShapeshifterAgrDsoClient",
+    "ShapeshifterCroAgrClient",
+    "ShapeshifterCroDsoClient",
+    "ShapeshifterDsoAgrClient",
+    "ShapeshifterDsoCroClient"
+]
+
+client_map = {
+    (client.sender_role, client.recipient_role): client
+    for client in [globals()[name] for name in __all__]
+}

shapeshifter-uftp-1.0.0/shapeshifter_uftp/client/agr_cro_client.py

@@ -0,0 +1,29 @@
+from .base_client import ShapeshifterClient
+from ..uftp import (
+    PayloadMessageResponse,
+    AgrPortfolioQuery,
+    AgrPortfolioUpdate,
+)
+
+
+class ShapeshifterAgrCroClient(ShapeshifterClient):
+    """
+    Client that allows the Aggregator to connect to the CRO.
+    """
+
+    sender_role = "AGR"
+    recipient_role = "CRO"
+
+    def send_agr_portfolio_update(self, message: AgrPortfolioUpdate) -> PayloadMessageResponse:
+        """
+        The AGRPortfolioUpdate is used by the AGR to indicate on which
+        Connections it represents prosumers.
+        """
+        return self._send_message(message)
+
+    def send_agr_portfolio_query(self, message: AgrPortfolioQuery) -> PayloadMessageResponse:
+        """
+        The AGRPortfolioQuery is used by the AGR to retrieve additional
+        information on the connections.
+        """
+        return self._send_message(message)

shapeshifter-uftp-1.0.0/shapeshifter_uftp/client/agr_dso_client.py

@@ -0,0 +1,92 @@
+from .base_client import ShapeshifterClient
+from ..uftp import (
+    DPrognosis,
+    FlexRequestResponse,
+    FlexOffer,
+    FlexOfferRevocation,
+    FlexOrderResponse,
+    FlexSettlementResponse,
+    FlexReservationUpdateResponse,
+    Metering,
+    PayloadMessageResponse
+)
+
+
+class ShapeshifterAgrDsoClient(ShapeshifterClient):
+    """
+    Client that allows the Aggregator to connect to the DSO.
+    """
+
+    sender_role = "AGR"
+    recipient_role = "DSO"
+
+    def send_d_prognosis(self, message: DPrognosis) -> PayloadMessageResponse:
+        """
+        D-Prognosis messages are used to communicate D-prognoses between AGRs
+        and DSOs. D-Prognosis messages always contain data for all ISPs for the
+        period they apply to, even if a prognosis is sent after the start of
+        the period, when one or more ISPs are already in the operate or
+        settlement phase. Receiving implementations should ignore the
+        information supplied for those ISPs.
+        """
+        return self._send_message(message)
+
+    def send_flex_request_response(self, message: FlexRequestResponse) -> PayloadMessageResponse:
+        """
+        Upon receiving and processing a FlexRequest message, the receiving
+        implementation must reply with a FlexRequestResponse, indicating
+        whether the flexibility request was processed successfully.
+        """
+        return self._send_message(message)
+
+    def send_flex_offer(self, message: FlexOffer) -> PayloadMessageResponse:
+        """
+        FlexOffer messages are used by AGRs to make DSOs an offer for provision
+        of flexibility. A FlexOffer message contains a list of ISPs and, for
+        each ISP, the change in consumption or production offered and the price
+        for the total amount of flexibility offered. FlexOffer messages can be
+        sent once a FlexRequest message has been received but can also be sent
+        unsolicited. Note that multiple FlexOffer messages may be sent based on
+        a single FlexRequest, e.g. to increase the chance that the DSO will
+        order at least part of its available flexibility. The AGR must make
+        sure that it can actually provide the flexibility offered across all of
+        its FlexOffers.
+        """
+        return self._send_message(message)
+
+    def send_flex_offer_revocation(self, message: FlexOfferRevocation) -> PayloadMessageResponse:
+        """
+        The FlexOfferRevocation message is used by the AGR to revoke a FlexOffer
+        previously sent to a DSO. It voids the FlexOffer, even if its validity
+        time has not yet expired. Revocation is not allowed for FlexOffers that
+        already have associated accepted FlexOrders.
+        """
+        return self._send_message(message)
+
+    def send_flex_order_response(self, message: FlexOrderResponse) -> PayloadMessageResponse:
+        """
+        Confirm the flex order.
+        """
+        return self._send_message(message)
+
+    def send_flex_settlement_response(self, message: FlexSettlementResponse) -> PayloadMessageResponse:
+        """
+        Upon receiving and processing a FlexSettlement message, the AGR must
+        reply with a FlexSettlementResponse, indicating whether the initial
+        message was handled successfully. When a FlexSettlement message is
+        rejected, the DSO should consider all FlexOrderSettlement elements of
+        that message related to potential dispute.
+        """
+        return self._send_message(message)
+
+    def send_flex_reservation_update_response(self, message: FlexReservationUpdateResponse) -> PayloadMessageResponse:
+        """
+        Confirm the flex reservation update.
+        """
+        return self._send_message(message)
+
+    def send_metering(self, message: Metering) -> PayloadMessageResponse:
+        """
+        Send metering data to the DSO.
+        """
+        return self._send_message(message)

shapeshifter-uftp-1.0.0/shapeshifter_uftp/client/base_client.py

@@ -0,0 +1,224 @@
+from datetime import datetime, timezone
+import sched
+import time
+from queue import Queue
+from threading import Thread, Event
+from uuid import uuid4
+import requests
+from ..uftp import PayloadMessage, PayloadMessageResponse, SignedMessage
+from ..logging import logger
+from ..exceptions import ClientTransportException
+from .. import transport
+
+
+class ShapeshifterClient():
+    """
+    Basis for all Shapeshifter client.
+    """
+
+    sender_role: str
+    recipient_role: str
+    protocol_version = "3.0.0"
+    num_outgoing_workers = 10
+    num_delivery_attempts = 10
+    request_timeout = 30
+    exponential_retry_factor = 1
+    exponential_retry_base = 2
+
+    def __init__(
+        self,
+        sender_domain: str,
+        signing_key: str,
+        recipient_domain: str,
+        recipient_endpoint: str = None,
+        recipient_signing_key: str = None,
+    ):
+        """
+        Shapeshifter client class that allows you to initiate messages to a different party.
+        :param sender_domain: your sender domain
+        :param signing_key:   your private signing key
+        :param recipient_domain:   the domain of the recipient
+        :param recipient_endpoint: the full http endpoint URL of the recipient. If omitted,
+                                   will look up the endpoint using DNS.
+        :recipient_signing_key:    the public signing key of the recipient. If omitted, will
+                                   look up the signing key using DNS.
+        """
+        if recipient_domain is None and recipient_endpoint is None:
+            raise ValueError(
+                "One of recipient_domain or recipient_endpoint must be provided."
+            )
+
+        self.sender_domain = sender_domain
+        self.signing_key = signing_key
+        self.recipient_domain = recipient_domain
+        self.recipient_endpoint = recipient_endpoint
+        self.recipient_signing_key = recipient_signing_key
+
+        # The outgoing queue and scheduler are used when queueing
+        # messages for delivery later. This allows the Shapeshifter
+        # UFTP client to handle message retries on an exponential
+        # time schedule, and delivers the result in the provided
+        # callback function.
+        self.outgoing_queue = Queue()
+        self.outgoing_workers = None
+        self.scheduler = sched.scheduler(time.monotonic, time.sleep)
+        self.scheduler_event = Event()
+        self.scheduler_thread = None
+
+    def _send_message(self, message: PayloadMessage) -> PayloadMessageResponse:
+        """
+        Perform an operation. This will take the message object, pack
+        it up into a SignedMessage, sign and seal it, and send it to
+        the recipient. It returns an unsealed PayloadMessageResponse
+        that contains the functional status of the request. The
+        actual response always arries asynchronously on your service
+        (which runs separately).
+        """
+        if not isinstance(message, PayloadMessage):
+            raise TypeError(
+                f"'message' must be a (subclass of) PayloadMessage, you provided: {type(message)}"
+            )
+
+        # Fill the PayloadMessage's fields that are common to all
+        # messages. We don't require the developer to fill these out
+        # every time they create any message, in order to reduce the
+        # duplicated code that would result in, and all of these
+        # properties can be calculated in the framework anyway.
+        message.version = self.protocol_version
+        message.sender_domain = self.sender_domain
+        message.sender_role = self.sender_role
+        message.recipient_domain = self.recipient_domain
+        message.time_stamp = message.time_stamp or datetime.now(timezone.utc).isoformat()
+        message.message_id = message.message_id or str(uuid4())
+        message.conversation_id = message.conversation_id or str(uuid4())
+
+        logger.info(f"The PayloadMessage is: {message}")
+
+        # Seal the message using our own private signing key
+        sealed_message = transport.seal_message(message, self.signing_key)
+
+        # Pack up the message into a SignedMessage
+        signed_message = SignedMessage(
+            sender_domain=self.sender_domain,
+            sender_role=self.sender_role,
+            body=sealed_message
+        )
+
+        # Serialize the message into an XML blob
+        serialized_message = transport.to_xml(signed_message)
+
+        logger.debug(f"Sending message to {self.recipient_endpoint}:")
+        logger.debug(serialized_message)
+
+        # Send the request to the relevant endpoint
+        response = requests.post(
+            self.recipient_endpoint,
+            data=serialized_message,
+            headers={"Content-Type": "text/xml; charset=utf-8"},
+            timeout=self.request_timeout,
+        )
+        if response.status_code != 200:
+            error_msg = (
+                f"Request to {self.recipient_endpoint} was not succesful: "
+                f"HTTP {response.status_code}: {response.text}"
+            )
+            logger.error(error_msg)
+            raise ClientTransportException(error_msg)
+
+        # Instantiate a SignedMessage object from the response bytes.
+        sealed_response = transport.parser.from_bytes(response.content)
+
+        # Use the static key that was provided when the client was
+        # initialized, or retrieve the key using DNS. The DNS
+        # implementation caches the DNS lookups so they are not that
+        # expensive after the first time.
+        if not (recipient_signing_key := self.recipient_signing_key):
+            recipient_signing_key = transport.get_key(self.recipient_domain, self.recipient_role)
+
+        # Unseal the response using the other party's public signing
+        # key and return the response message within it
+        return transport.unseal_message(
+            message=sealed_response.body,
+            public_key=recipient_signing_key
+        )
+
+
+    # ------------------------------------------------------------ #
+    #     Methods related to queueing and scheduling outgoing      #
+    #                          messages.                           #
+    # ------------------------------------------------------------ #
+
+    def _queue_message(self, message, callback, attempt=1):
+        self.outgoing_queue.put((message, callback, attempt))
+        self._run_outgoing_workers()
+
+    def _outgoing_worker(self):
+        while True:
+            message, callback, attempt = self.outgoing_queue.get()
+            try:
+                response = self._send_message(message)
+            except Exception as exc:
+                if attempt <= self.num_delivery_attempts:
+                    # Reschedule with exponential backoff
+                    delay_time = self.exponential_retry_factor * self.exponential_retry_base ** attempt
+                    logger.warning(
+                        f"Outgoing message {message.__class__.__name__} to "
+                        f"{message.recipient_domain} could not be delivered "
+                        f"due to a {exc.__class__.__name__}, will try again in {delay_time:.0f} seconds."
+                    )
+                    self.scheduler.enter(
+                        delay=delay_time,
+                        priority=1,
+                        action=self._queue_message,
+                        argument=((message, callback, attempt + 1))
+                    )
+                    self._run_scheduler()
+                else:
+                    logger.error(
+                        f"Could not deliver {message.__class__.__name__} "
+                        f"to {self.recipient_role} at {self.recipient_domain}, "
+                        f"even after {self.num_delivery_attempts} attempts."
+                    )
+            else:
+                try:
+                    callback(response)
+                except Exception as err:
+                    logger.error("There was an exception during the callback "
+                                 f"for a {message.__class__.__name__} message: "
+                                 f"{err.__class__.__name__}: {err}")
+            finally:
+                self.outgoing_queue.task_done()
+
+    def _run_scheduler(self):
+        """
+        Make sure the scheduler thread is running and awake.
+        """
+        if not self.scheduler_thread:
+            self.scheduler_thread = Thread(target=self._scheduler_thread, daemon=True)
+            self.scheduler_thread.start()
+        self.scheduler_event.set()
+
+    def _scheduler_thread(self):
+        """
+        Intended to run the python scheduler in a background thread.
+        You can wake it up anytime by setting the scheduler event.
+        """
+        while True:
+            self.scheduler_event.wait()
+            self.scheduler_event.clear()
+            self.scheduler.run()
+
+    def _run_outgoing_workers(self):
+        """
+        Start up the outgoing queue workers.
+        """
+        if not self.outgoing_workers:
+            self.outgoing_workers = [Thread(target=self._outgoing_worker, daemon=True) for _ in range(self.num_outgoing_workers)]
+            for thread in self.outgoing_workers:
+                thread.start()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        pass

shapeshifter-uftp-1.0.0/shapeshifter_uftp/client/cro_agr_client.py

@@ -0,0 +1,25 @@
+from .base_client import ShapeshifterClient
+from ..uftp import AgrPortfolioUpdateResponse, AgrPortfolioQueryResponse
+
+
+class ShapeshifterCroAgrClient(ShapeshifterClient):
+    """
+    Client that allows the CRO to connect to the Aggregator.
+    """
+
+    sender_role = "CRO"
+    recipient_role = "AGR"
+
+    def send_agr_portfolio_update_response(self, message: AgrPortfolioUpdateResponse):
+        """
+        The DSOPortfolioUpdate is used by the DSO to indicate on which
+        congestion points it wants to engage in flexibility trading.
+        """
+        return self._send_message(message)
+
+    def send_agr_portfolio_query_response(self, message: AgrPortfolioQueryResponse):
+        """
+        DSOPortfolioQuery is used by DSOs to discover which AGRs represent
+        connections on its registered congestion point(s).
+        """
+        return self._send_message(message)