dtps-http 1.4.6__tar.gz

dtps_http-1.4.6/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.4
+Name: dtps-http
+Version: 1.4.6
+Summary: 
+License-File: LICENSE.pdf
+Author: Andrea Censi
+Author-email: AndreaCensi@users.noreply.github.com
+Requires-Python: >=3.8,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: aiohttp (>=3,<4)
+Requires-Dist: aiopubsub (>=3,<4)
+Requires-Dist: cbor2 (>=5,<6)
+Requires-Dist: jsonpatch (>=1.33,<2.0)
+Requires-Dist: methodtools (>=0.4,<0.5)
+Requires-Dist: prettyprinter (>=0.18.0,<0.19.0)
+Requires-Dist: psutil (>=7,<8)
+Requires-Dist: pydantic (==2.10.6)
+Requires-Dist: pymdown-extensions (>=10,<11)
+Requires-Dist: tcp_latency (==0.0.12)
+Requires-Dist: timeout-decorator (>=0.5.0,<0.6.0)
+Requires-Dist: typing_extensions (>=4,<5)
+Requires-Dist: urllib3 (>=2,<3)
+Requires-Dist: xxhash (>=3.4.1,<4.0.0)

dtps_http-1.4.6/pyproject.toml

@@ -0,0 +1,68 @@
+[build-system]
+requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
+build-backend = "poetry_dynamic_versioning.backend"
+
+
+[tool]
+[tool.poetry]
+name = "dtps-http"
+version = "1.4.6"
+description = ""
+authors = ["Andrea Censi <AndreaCensi@users.noreply.github.com>"]
+packages = [
+    { from = "src", include = "dtps" },
+    { from = "src", include = "dtps_http" },
+    { from = "src", include = "dtps_http_programs" },
+]
+
+include = [
+    { path = "src/dtps_http/static/**", format = ['sdist', 'wheel'] }
+]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+aiohttp = "^3"
+cbor2 = "^5"
+tcp_latency = "0.0.12"
+typing_extensions = "^4"
+pymdown-extensions = "^10"
+aiopubsub = "^3"
+methodtools = "^0.4"
+pydantic = "2.10.6"
+psutil = "^7"
+urllib3 = "^2"
+jsonpatch = "^1.33"
+
+
+# other dependencies: aiohttp-devtools pre-commit mkdocs mkdocs-material mkdocstrings[python]
+# pip install "mkdocstrings[python]"  mkdocs-material
+timeout-decorator = "^0.5.0"
+prettyprinter = "^0.18.0"
+xxhash = "^3.4.1"
+
+[tool.poetry.scripts]
+
+dtps-http-py-server-example-clock = 'dtps_http_programs:clock_main'
+dtps-http-py-server = 'dtps_http_programs:server_main'
+dtps-http-py-client-stats = 'dtps_http_programs:dtps_stats_main'
+dtps-http-py-proxy = 'dtps_http_programs:dtps_proxy_main'
+dtps-http-py-send-continuous = 'dtps_http_programs:dtps_send_continuous_main'
+dtps-http-py-listen= 'dtps_http_programs:dtps_listen_main'
+dtps-http-py= 'dtps_http_programs:dtps_main'
+
+
+[tool.poetry-dynamic-versioning]
+enable = false
+dirty = false
+metadata = false
+tagged-metadata = false
+vcs = "git"
+style = "pep440"
+strict = false
+
+
+
+[tool.poetry-dynamic-versioning.substitution]
+folders = [
+    { path = "src" }
+]

dtps_http-1.4.6/src/dtps/__init__.py

@@ -0,0 +1,13 @@
+__version__ = "1.4.6"
+
+from logging import DEBUG, getLogger
+
+logger = getLogger(__name__)
+logger.setLevel(DEBUG)
+
+from .config import *
+from .ergo_ui import *
+from .dtps_utils import *
+from dtps_http import RawData, TransformError
+
+_ = RawData, TransformError

dtps_http-1.4.6/src/dtps/config.py

@@ -0,0 +1,235 @@
+import os
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from typing import (
+    AsyncContextManager,
+    AsyncIterator,
+    cast,
+    ClassVar,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    TYPE_CHECKING,
+)
+
+from dtps_http import parse_url_unescape, ServerWrapped, URLString
+from . import logger
+from .ergo_ui import DTPSContext
+
+__all__ = [
+    "context",
+    "context_cleanup",
+]
+
+
+async def context(
+    base_name: str = "self", environment: Optional[Mapping[str, str]] = None, urls: Optional[List[str]] = None
+) -> "DTPSContext":
+    """
+    Initialize a DTPS interface from the environment from a given base name.
+
+    base_name is case-insensitive.
+
+    Environment variables of the form DTPS_BASE_<base_name> are used to get the info needed.
+
+    For example:
+
+        DTPS_BASE_SELF = "http://localhost:2120/" # use an existing server
+        DTPS_BASE_SELF = "http+unix://[socket]/" # use an existing unix socket
+
+    We can also use the special prefix "create:" to create a new server.
+    For example:
+
+        DTPS_BASE_SELF = "create:http://localhost:2120/" # create a new server
+
+    Moreover, we can use more than one base name, by adding a number at the end:
+
+        DTPS_BASE_SELF_0 = "create:http://localhost:2120/" #
+        DTPS_BASE_SELF_1 = "create:http+unix://[socket]/"
+
+
+    You need to call context.aclose() at the end to clean up resources.
+
+
+        c = context("mio", environment={'DTPS_BASE_MIO': url})
+
+    """
+    base_name = base_name.lower()
+
+    if environment is not None and urls is not None:
+        raise ValueError("You cannot create a context while passing both 'environment' and 'urls'")
+
+    if urls:
+        environment = environment_from_urls(base_name, urls)
+
+    if environment is None:
+        if base_name in ContextManager.instances:
+            return ContextManager.instances[base_name].get_context()
+        else:
+            context_manager = await create_context(base_name, environment)
+            ContextManager.instances[base_name] = context_manager
+            return context_manager.get_context()
+    else:
+        context_manager = await create_context(base_name, environment)
+        return context_manager.get_context()
+
+
+if TYPE_CHECKING:
+
+    def context_cleanup(
+        base_name: str = "self", environment: Optional[Mapping[str, str]] = None
+    ) -> AsyncContextManager[DTPSContext]: ...
+
+else:
+
+    @asynccontextmanager
+    async def context_cleanup(
+        base_name: str = "self", environment: Optional[Mapping[str, str]] = None
+    ) -> AsyncIterator[DTPSContext]:
+        """Context manager to open a context and clean-up later."""
+        c = await context(base_name, environment)
+        try:
+            yield c
+        finally:
+            await c.aclose()
+
+
+async def create_context(base_name: str, environment: Optional[Mapping[str, str]]) -> "ContextManager":
+    contexts = get_context_info(environment)
+    if base_name not in contexts.contexts:
+        msg = f'Cannot find context "{base_name}" among {list(contexts.contexts)}'
+        raise KeyError(msg)
+
+    context_info = contexts.contexts[base_name]
+    logger.debug(f'Creating context "{base_name}" with {context_info} for environment {environment}')
+    return await ContextManager.create(base_name, context_info)
+
+
+class ContextManager:
+    instances: ClassVar[Dict[str, "ContextManager"]] = {}
+
+    context_info: "ContextInfo"
+
+    dtps_server_wrap: Optional[ServerWrapped]
+
+    @classmethod
+    async def create(cls, base_name: str, context_info: "ContextInfo") -> "ContextManager":
+        # if base_name in cls.instances:
+        #     msg = f'Context "{base_name}" already exists'
+        #     raise KeyError(msg)
+
+        # logger.info(f'Creating context "{base_name}" with {context_info}')
+
+        if context_info.is_create():
+            from .ergo_create import ContextManagerCreate
+
+            cm = ContextManagerCreate(base_name, context_info)
+        else:
+            from .ergo_use import ContextManagerUse
+
+            cm = ContextManagerUse(base_name, context_info)
+
+        # cls.instances[base_name] = cm
+        await cm.init()
+        return cm
+
+    def get_context(self) -> "DTPSContext":
+        raise NotImplementedError()
+
+
+@dataclass
+class ContextUrl:
+    url: URLString
+    create: bool
+
+    def __post_init__(self) -> None:
+        parse_url_unescape(self.url)
+
+
+@dataclass
+class ContextInfo:
+    urls: List[ContextUrl]
+
+    def is_create(self) -> bool:
+        return all(x.create for x in self.urls)
+
+    def get_tcp_and_unix(self) -> Tuple[List[Tuple[str, int]], List[str]]:
+        tcp: List[Tuple[str, int]] = []
+        unix: List[str] = []
+        for u in self.urls:
+            url_ = parse_url_unescape(u.url)
+            if url_.scheme == "http+unix":
+                host = url_.host
+                unix.append(host)
+
+            elif url_.scheme == "http" or url_.scheme == "https":
+                # rest = url.url[len("http://") :]
+                # host, _, rest = rest.partition("/")
+                # host, _, port = host.partition(":")
+                # port = int(port)
+                host = url_.host or "localhost"
+                if not url_.port:
+                    port = 0
+                elif isinstance(url_.port, str):
+                    port = int(url_.port)
+                else:
+                    port = url_.port
+
+                tcp.append((host, port))
+            else:
+                msg = f'Invalid url "{url_}". Must start with "http://" or "http+unix://".'
+                raise ValueError(msg)
+        return tcp, unix
+
+
+@dataclass
+class ContextsInfo:
+    contexts: Dict[str, ContextInfo]
+
+
+BASE = "DTPS_BASE_"
+
+
+def get_context_info(environment: Optional[Mapping[str, str]]) -> ContextsInfo:
+    if environment is None:
+        environment = dict(os.environ)
+
+    contexts: Dict[str, ContextInfo] = {}
+    for k, v in environment.items():
+        if not k.startswith(BASE):
+            continue
+        rest = k[len(BASE) :]
+
+        name, _, rest = rest.partition("_")
+
+        name = name.lower()
+        if name not in contexts:
+            contexts[name] = ContextInfo(urls=[])
+
+        if v.startswith("create:"):
+            x = cast(URLString, v[len("create:") :])
+            create = True
+
+        else:
+            x = cast(URLString, v)
+            create = False
+        try:
+            parse_url_unescape(x)
+        except ValueError as e:
+            msg = f"Invalid url given by environment:\n{k} = {v}\nExtracted url: {x}"
+            raise ValueError(msg) from e
+        contexts[name].urls.append(ContextUrl(url=x, create=create))
+
+    for name, info in contexts.items():
+        all_create = all(x.create for x in info.urls)
+        all_not_create = all(not x.create for x in info.urls)
+        if not all_create and not all_not_create:
+            msg = f'Invalid context "{name}". All urls must be either "create:" or not.'
+            raise ValueError(msg)
+    return ContextsInfo(contexts=contexts)
+
+
+def environment_from_urls(name: str, urls: List[str]):
+    return {f"{BASE}{name}_{i}": url for i, url in enumerate(urls)}

dtps_http-1.4.6/src/dtps/dtps_utils.py

@@ -0,0 +1,121 @@
+import asyncio
+from asyncio import CancelledError, Queue
+from typing import Any, Awaitable, Callable, List, Optional, TypeVar
+
+from dtps_http import RawData
+from . import logger
+from .ergo_ui import DTPSContext, SubscriptionInterface
+
+__all__ = [
+    "process_lowdatasize_last_recent",
+]
+
+
+async def process_lowdatasize_last_recent(
+    context: DTPSContext,
+    expensive_callback: Callable[[RawData], Awaitable[None]],
+) -> "ExpensiveCallbackSubscription":
+    """
+    Make sure we are not a slow reader even for an expensive callback.
+
+    Suitable for this case:
+
+    0) only the last message is important (e.g. joystick)
+    1) there are a lot of small messages
+    2) low-latency is important
+    2) the callback is expensive (e.g. write to hardware)
+
+
+    Note! If the callback uses *blocking IO* we need to do it
+    in a different process! (or thread)
+
+    """
+    s = ExpensiveCallbackSubscription(expensive_callback)
+    await s.init(context)
+    return s
+
+
+X = TypeVar("X")
+
+
+async def queue_get_multiple(q: "Queue[X]") -> List[X]:
+    """Gets a packet of ready messages from the queue (at least one)."""
+    first = await q.get()
+    msgs = [first]
+    while True:
+        try:
+            msgs.append(q.get_nowait())
+        except asyncio.QueueEmpty:
+            break
+    return msgs
+
+
+class ExpensiveCallbackSubscription(SubscriptionInterface):
+    q: "Queue[RawData]"
+    sub: Optional[SubscriptionInterface]
+    task: "Optional[asyncio.Task[Any]]"
+
+    def __init__(self, expensive_callback: Callable[[RawData], Awaitable[None]]):
+        self.expensive_callback = expensive_callback
+        self.q = Queue()
+        self.sub = None
+        self.nreceived = 0
+        self.nprocessed = 0
+        self.nskipped = 0
+
+    async def init(self, context: DTPSContext) -> None:
+        self.sub = await context.subscribe(self.raw_callback)
+        self.task = asyncio.create_task(self.process_task())
+
+    async def process_task(self) -> None:
+        while True:
+            rd = await queue_get_multiple(self.q)
+            # only process the last one
+            last = rd[-1]
+            nskipped_now = len(rd) - 1
+            self.nprocessed += 1
+            self.nskipped += nskipped_now
+
+            if nskipped_now > 0:
+                percentage_skipped = self.nskipped / self.nreceived
+                logger.debug(
+                    f"Skipped {nskipped_now} messages now in this iteration.\n"
+                    f"Total received: {self.nreceived}. "
+                    f"Total skipped: {self.nskipped} ({percentage_skipped:.2%})\n"
+                )
+            try:
+                await self.expensive_callback(last)
+            except CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Error in expensive callback", exc_info=e)
+                continue
+
+    async def raw_callback(self, r: RawData) -> None:
+        self.nreceived += 1
+        self.q.put_nowait(r)
+
+    async def aclose(self) -> None:
+        if self.nreceived == 0:
+            logger.debug("No messages received")
+        else:
+
+            percentage_skipped = self.nskipped / self.nreceived
+            percentage_processed = self.nprocessed / self.nreceived
+
+            logger.debug(
+                f"Final statistics:\n"
+                f" Total received:  {self.nreceived:5} \n"
+                f" Total processed: {self.nprocessed:5} ({percentage_processed:.2%})\n"
+                f" Total skipped:   {self.nskipped:5} ({percentage_skipped:.2%})\n"
+            )
+
+        if self.sub is not None:
+            await self.sub.unsubscribe()
+            self.sub = None
+        if self.task is not None:
+            self.task.cancel()
+            self.task = None
+
+    async def unsubscribe(self) -> None:
+        await self.aclose()