refract-io-lib 0.1.0__tar.gz

refract_io_lib-0.1.0/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.3
+Name: refract-io-lib
+Version: 0.1.0
+Summary: Refract I/O library
+Author: Richard Chen
+Author-email: Richard Chen <richardchen.93@gmail.com>
+Requires-Dist: grpcio>=1.60.0
+Requires-Dist: protobuf>=4.25.0
+Requires-Dist: zeroconf>=0.131.0
+Requires-Python: >=3.10
+Project-URL: Homepage, https://refractvisualizer.ca/

refract_io_lib-0.1.0/pyproject.toml

@@ -0,0 +1,74 @@
+[project]
+name = "refract-io-lib"
+version = "0.1.0"
+description = "Refract I/O library"
+authors = [
+    { name = "Richard Chen", email = "richardchen.93@gmail.com" }
+]
+requires-python = ">=3.10"
+dependencies = [
+    "grpcio>=1.60.0",
+    "protobuf>=4.25.0",
+    "zeroconf>=0.131.0",
+]
+
+[project.urls]
+Homepage = "https://refractvisualizer.ca/"
+
+[dependency-groups]
+dev = [
+    "black>=25.1.0",
+    "grpc-stubs>=1.53.0",
+    "grpcio-tools>=1.60.0",
+    "isort>=6.0.0",
+    "mypy>=1.15.0",
+    "pre-commit>=4.0.0",
+    "ruff>=0.11.0",
+    "types-protobuf>=4.25.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.9.21,<0.10.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "refract_io"
+
+[tool.black]
+line-length = 88
+target-version = ["py310"]
+extend-exclude = "src/refract_io/_proto"
+
+[tool.isort]
+profile = "black"
+line_length = 88
+src_paths = ["src"]
+extend_skip_glob = ["src/refract_io/_proto/*"]
+
+[tool.ruff]
+line-length = 88
+target-version = "py310"
+src = ["src"]
+extend-exclude = ["src/refract_io/_proto"]
+
+[tool.ruff.lint]
+# isort runs in pre-commit; ruff lint omits "I" to avoid conflicting import rules.
+select = ["E", "W", "F", "B", "UP", "C4"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+mypy_path = "src"
+explicit_package_bases = true
+packages = ["refract_io"]
+
+[[tool.mypy.overrides]]
+module = "refract_io._proto.*"
+ignore_errors = true
+follow_imports = "skip"
+
+[[tool.mypy.overrides]]
+module = "zeroconf"
+ignore_missing_imports = true

refract_io_lib-0.1.0/src/refract_io/__init__.py

@@ -0,0 +1,30 @@
+from refract_io._client import RefractStream
+from refract_io._discovery import discover_refract
+from refract_io._types import ValueType
+
+float32 = ValueType.FLOAT32
+float64 = ValueType.FLOAT64
+int8 = ValueType.INT8
+int16 = ValueType.INT16
+int32 = ValueType.INT32
+int64 = ValueType.INT64
+uint8 = ValueType.UINT8
+uint16 = ValueType.UINT16
+uint32 = ValueType.UINT32
+uint64 = ValueType.UINT64
+
+__all__ = [
+    "RefractStream",
+    "ValueType",
+    "discover_refract",
+    "float32",
+    "float64",
+    "int8",
+    "int16",
+    "int32",
+    "int64",
+    "uint8",
+    "uint16",
+    "uint32",
+    "uint64",
+]

refract_io_lib-0.1.0/src/refract_io/_client.py

@@ -0,0 +1,269 @@
+from __future__ import annotations
+
+import logging
+import queue
+import struct
+import threading
+from collections.abc import Generator, Sequence
+from types import TracebackType
+from typing import Any
+
+import grpc
+
+from refract_io._discovery import discover_refract
+from refract_io._proto import kvstream_pb2 as _pb2
+from refract_io._proto import kvstream_pb2_grpc as _pb2_grpc
+from refract_io._types import DTYPE_INFO, ValueType
+
+# Re-bind as Any so mypy doesn't complain about dynamic proto attributes
+kvstream_pb2: Any = _pb2
+kvstream_pb2_grpc: Any = _pb2_grpc
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_HOST = "localhost"
+_DEFAULT_PORT = 50051
+
+# Sentinel used to signal the background thread to stop.
+_STOP = object()
+
+
+class RefractStream:
+    """Stream tabular data over gRPC to the Refract app.
+
+    ``host`` and ``port`` are optional.  When both are ``None`` the client
+    attempts Bonjour autodiscovery and falls back to ``localhost:50051``.
+    ``port`` accepts either an ``int`` or a numeric ``str``.
+    """
+
+    def __init__(
+        self,
+        host: str | None = None,
+        port: int | str | None = None,
+    ) -> None:
+        resolved_host, resolved_port = self._resolve_address(host, port)
+        self._host: str = resolved_host
+        self._port: int = resolved_port
+
+        self._channel: grpc.Channel | None = None
+        self._stub: kvstream_pb2_grpc.KVStreamStub | None = None
+        self._queue: queue.Queue[object] = queue.Queue()
+        self._thread: threading.Thread | None = None
+        self._running: bool = False
+
+        self._lock = threading.Lock()
+        # table_id -> list of (struct_fmt, byte_size) per column
+        self._tables: dict[int, list[tuple[str, int]]] = {}
+        self._pending_registrations: list[kvstream_pb2.StreamMessage] = []
+
+    # ------------------------------------------------------------------
+    # Address resolution
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _resolve_address(
+        host: str | None,
+        port: int | str | None,
+    ) -> tuple[str, int]:
+        if host is not None or port is not None:
+            return (
+                host if host is not None else _DEFAULT_HOST,
+                int(port) if port is not None else _DEFAULT_PORT,
+            )
+        # Autodiscover
+        result = discover_refract()
+        if result is not None:
+            logger.info("Autodiscovered Refract at %s:%d", *result)
+            return result
+        logger.info(
+            "No Refract service found via Bonjour, " "falling back to %s:%d",
+            _DEFAULT_HOST,
+            _DEFAULT_PORT,
+        )
+        return (_DEFAULT_HOST, _DEFAULT_PORT)
+
+    # ------------------------------------------------------------------
+    # Table registration
+    # ------------------------------------------------------------------
+
+    def create_table(
+        self,
+        id: int,
+        name: str,
+        columns: dict[str, ValueType],
+    ) -> None:
+        """Register a table schema.
+
+        ``columns`` maps column names to :class:`ValueType` constants
+        (e.g. ``{'timestamp': refract_io.float64, ...}``).
+        """
+        col_defs: list[kvstream_pb2.ColumnDef] = []
+        fmt_parts: list[tuple[str, int]] = []
+        for col_name, vtype in columns.items():
+            vtype = ValueType(vtype)  # validate
+            fmt, size = DTYPE_INFO[vtype]
+            col_defs.append(
+                kvstream_pb2.ColumnDef(name=col_name, value_type=int(vtype))
+            )
+            fmt_parts.append((fmt, size))
+
+        with self._lock:
+            self._tables[id] = fmt_parts
+
+        msg = kvstream_pb2.StreamMessage(
+            register_table=kvstream_pb2.RegisterTable(
+                table_id=id,
+                name=name,
+                columns=col_defs,
+            )
+        )
+        if self._running:
+            self._queue.put(msg)
+        else:
+            self._pending_registrations.append(msg)
+
+    # ------------------------------------------------------------------
+    # Sending rows
+    # ------------------------------------------------------------------
+
+    def send_row(self, table_id: int, values: Sequence[int | float]) -> None:
+        """Send a single row for *table_id*.
+
+        Values are packed in registration order using the column dtypes.
+        Auto-starts the connection if not already running.
+        """
+        self._ensure_started()
+
+        with self._lock:
+            fmt_parts = self._tables.get(table_id)
+        if fmt_parts is None:
+            raise ValueError(f"Table {table_id} not registered")
+        if len(values) != len(fmt_parts):
+            raise ValueError(f"Expected {len(fmt_parts)} values, got {len(values)}")
+
+        packed = b"".join(
+            struct.pack(fmt, val)
+            for val, (fmt, _) in zip(values, fmt_parts, strict=True)
+        )
+        msg = kvstream_pb2.StreamMessage(
+            table_row=kvstream_pb2.TableRow(
+                table_id=table_id,
+                values=packed,
+            )
+        )
+        self._queue.put(msg)
+
+    def send_rows(
+        self,
+        table_id: int,
+        rows: Sequence[Sequence[int | float]],
+    ) -> None:
+        """Send multiple rows for *table_id* in one call."""
+        self._ensure_started()
+
+        with self._lock:
+            fmt_parts = self._tables.get(table_id)
+        if fmt_parts is None:
+            raise ValueError(f"Table {table_id} not registered")
+
+        expected = len(fmt_parts)
+        for row in rows:
+            if len(row) != expected:
+                raise ValueError(f"Expected {expected} values per row, got {len(row)}")
+            packed = b"".join(
+                struct.pack(fmt, val)
+                for val, (fmt, _) in zip(row, fmt_parts, strict=True)
+            )
+            msg = kvstream_pb2.StreamMessage(
+                table_row=kvstream_pb2.TableRow(
+                    table_id=table_id,
+                    values=packed,
+                )
+            )
+            self._queue.put(msg)
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def start(self) -> None:
+        """Open the gRPC channel and start the background streaming thread."""
+        if self._running:
+            return
+        self._running = True
+        self._channel = grpc.insecure_channel(f"{self._host}:{self._port}")
+        self._stub = kvstream_pb2_grpc.KVStreamStub(self._channel)
+        self._thread = threading.Thread(
+            target=self._stream_loop, daemon=True, name="refract-stream"
+        )
+        self._thread.start()
+
+    def close(self) -> None:
+        """Stop the background thread and close the gRPC channel."""
+        if not self._running:
+            return
+        self._running = False
+        self._queue.put(_STOP)
+        if self._thread is not None:
+            self._thread.join(timeout=5)
+            self._thread = None
+        if self._channel is not None:
+            self._channel.close()
+            self._channel = None
+
+    stop = close
+
+    def _ensure_started(self) -> None:
+        if not self._running:
+            self.start()
+
+    # ------------------------------------------------------------------
+    # Context manager
+    # ------------------------------------------------------------------
+
+    def __enter__(self) -> RefractStream:
+        self.start()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    # ------------------------------------------------------------------
+    # Background streaming
+    # ------------------------------------------------------------------
+
+    def _message_generator(
+        self,
+    ) -> Generator[kvstream_pb2.StreamMessage, None, None]:
+        # Send pending registrations first
+        yield from self._pending_registrations
+        self._pending_registrations.clear()
+
+        # Stream from queue
+        while self._running:
+            try:
+                item = self._queue.get(timeout=0.1)
+                if item is _STOP:
+                    break
+                assert isinstance(item, kvstream_pb2.StreamMessage)  # noqa: S101
+                yield item
+            except queue.Empty:
+                continue
+
+    def _stream_loop(self) -> None:
+        try:
+            assert self._stub is not None  # noqa: S101
+            response = self._stub.Stream(self._message_generator())
+            if not response.ok:
+                logger.error("Server error: %s", response.error)
+        except grpc.RpcError as exc:
+            if self._running:
+                logger.error("gRPC error: %s", exc)
+        except Exception:
+            if self._running:
+                logger.exception("Unexpected error in stream loop")

refract_io_lib-0.1.0/src/refract_io/_discovery.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import logging
+import socket
+import threading
+
+from zeroconf import ServiceBrowser, ServiceStateChange, Zeroconf
+
+logger = logging.getLogger(__name__)
+
+SERVICE_TYPE = "_refract-stream._tcp.local."
+
+
+def discover_refract(timeout: float = 3.0) -> tuple[str, int] | None:
+    """Browse for a Refract Stream service advertised via Bonjour.
+
+    Returns ``(host, port)`` of the first instance found, or ``None``
+    if no service is discovered within *timeout* seconds.
+    """
+    result: tuple[str, int] | None = None
+    found = threading.Event()
+
+    def on_state_change(
+        zc: Zeroconf,
+        service_type: str,
+        name: str,
+        state_change: ServiceStateChange,
+    ) -> None:
+        nonlocal result
+        if state_change is not ServiceStateChange.Added:
+            return
+        info = zc.get_service_info(service_type, name)
+        if info is None:
+            return
+        addresses = info.parsed_addresses()
+        if not addresses or info.port is None:
+            return
+        port = info.port
+        # Prefer non-link-local IPv4 addresses
+        host = addresses[0]
+        for addr in addresses:
+            try:
+                packed = socket.inet_aton(addr)
+                if packed[0:2] != b"\xa9\xfe":  # not 169.254.x.x
+                    host = addr
+                    break
+            except OSError:
+                continue
+        result = (host, port)
+        logger.debug("Discovered Refract at %s:%d", host, port)
+        found.set()
+
+    zc = Zeroconf()
+    try:
+        ServiceBrowser(zc, SERVICE_TYPE, handlers=[on_state_change])
+        found.wait(timeout=timeout)
+    finally:
+        zc.close()
+
+    return result

refract_io_lib-0.1.0/src/refract_io/_proto/kvstream_pb2.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: kvstream.proto
+# Protobuf Python Version: 6.31.1
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    6,
+    31,
+    1,
+    '',
+    'kvstream.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0ekvstream.proto\x12\x10refract.kvstream\"J\n\tColumnDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.refract.kvstream.ValueType\"]\n\rRegisterTable\x12\x10\n\x08table_id\x18\x01 \x01(\r\x12\x0c\n\x04name\x18\x02 \x01(\t\x12,\n\x07\x63olumns\x18\x03 \x03(\x0b\x32\x1b.refract.kvstream.ColumnDef\",\n\x08TableRow\x12\x10\n\x08table_id\x18\x01 \x01(\r\x12\x0e\n\x06values\x18\x02 \x01(\x0c\"\x86\x01\n\rStreamMessage\x12\x39\n\x0eregister_table\x18\x01 \x01(\x0b\x32\x1f.refract.kvstream.RegisterTableH\x00\x12/\n\ttable_row\x18\x02 \x01(\x0b\x32\x1a.refract.kvstream.TableRowH\x00\x42\t\n\x07payload\"+\n\x0eStreamResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t*\x7f\n\tValueType\x12\x0b\n\x07\x46LOAT32\x10\x00\x12\x0b\n\x07\x46LOAT64\x10\x01\x12\x08\n\x04INT8\x10\x02\x12\t\n\x05INT16\x10\x03\x12\t\n\x05INT32\x10\x04\x12\t\n\x05INT64\x10\x05\x12\t\n\x05UINT8\x10\x06\x12\n\n\x06UINT16\x10\x07\x12\n\n\x06UINT32\x10\x08\x12\n\n\x06UINT64\x10\t2Y\n\x08KVStream\x12M\n\x06Stream\x12\x1f.refract.kvstream.StreamMessage\x1a .refract.kvstream.StreamResponse(\x01\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'kvstream_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  DESCRIPTOR._loaded_options = None
+  _globals['_VALUETYPE']._serialized_start=435
+  _globals['_VALUETYPE']._serialized_end=562
+  _globals['_COLUMNDEF']._serialized_start=36
+  _globals['_COLUMNDEF']._serialized_end=110
+  _globals['_REGISTERTABLE']._serialized_start=112
+  _globals['_REGISTERTABLE']._serialized_end=205
+  _globals['_TABLEROW']._serialized_start=207
+  _globals['_TABLEROW']._serialized_end=251
+  _globals['_STREAMMESSAGE']._serialized_start=254
+  _globals['_STREAMMESSAGE']._serialized_end=388
+  _globals['_STREAMRESPONSE']._serialized_start=390
+  _globals['_STREAMRESPONSE']._serialized_end=433
+  _globals['_KVSTREAM']._serialized_start=564
+  _globals['_KVSTREAM']._serialized_end=653
+# @@protoc_insertion_point(module_scope)

refract_io_lib-0.1.0/src/refract_io/_proto/kvstream_pb2_grpc.py

@@ -0,0 +1,97 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+import warnings
+
+from . import kvstream_pb2 as kvstream__pb2
+
+GRPC_GENERATED_VERSION = '1.80.0'
+GRPC_VERSION = grpc.__version__
+_version_not_supported = False
+
+try:
+    from grpc._utilities import first_version_is_lower
+    _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION)
+except ImportError:
+    _version_not_supported = True
+
+if _version_not_supported:
+    raise RuntimeError(
+        f'The grpc package installed is at version {GRPC_VERSION},'
+        + ' but the generated code in kvstream_pb2_grpc.py depends on'
+        + f' grpcio>={GRPC_GENERATED_VERSION}.'
+        + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
+        + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'
+    )
+
+
+class KVStreamStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Stream = channel.stream_unary(
+                '/refract.kvstream.KVStream/Stream',
+                request_serializer=kvstream__pb2.StreamMessage.SerializeToString,
+                response_deserializer=kvstream__pb2.StreamResponse.FromString,
+                _registered_method=True)
+
+
+class KVStreamServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def Stream(self, request_iterator, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_KVStreamServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'Stream': grpc.stream_unary_rpc_method_handler(
+                    servicer.Stream,
+                    request_deserializer=kvstream__pb2.StreamMessage.FromString,
+                    response_serializer=kvstream__pb2.StreamResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'refract.kvstream.KVStream', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+    server.add_registered_method_handlers('refract.kvstream.KVStream', rpc_method_handlers)
+
+
+ # This class is part of an EXPERIMENTAL API.
+class KVStream(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def Stream(request_iterator,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.stream_unary(
+            request_iterator,
+            target,
+            '/refract.kvstream.KVStream/Stream',
+            kvstream__pb2.StreamMessage.SerializeToString,
+            kvstream__pb2.StreamResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)

refract_io_lib-0.1.0/src/refract_io/_types.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from enum import IntEnum
+
+
+class ValueType(IntEnum):
+    """Column data types matching the kvstream protobuf ValueType enum."""
+
+    FLOAT32 = 0
+    FLOAT64 = 1
+    INT8 = 2
+    INT16 = 3
+    INT32 = 4
+    INT64 = 5
+    UINT8 = 6
+    UINT16 = 7
+    UINT32 = 8
+    UINT64 = 9
+
+
+# Internal mapping: ValueType -> (struct format, byte size)
+DTYPE_INFO: dict[ValueType, tuple[str, int]] = {
+    ValueType.FLOAT32: ("<f", 4),
+    ValueType.FLOAT64: ("<d", 8),
+    ValueType.INT8: ("<b", 1),
+    ValueType.INT16: ("<h", 2),
+    ValueType.INT32: ("<i", 4),
+    ValueType.INT64: ("<q", 8),
+    ValueType.UINT8: ("<B", 1),
+    ValueType.UINT16: ("<H", 2),
+    ValueType.UINT32: ("<I", 4),
+    ValueType.UINT64: ("<Q", 8),
+}