intersect-sdk-common 0.0.0a0__tar.gz

intersect_sdk_common-0.0.0a0/PKG-INFO

@@ -0,0 +1,52 @@
+Metadata-Version: 2.3
+Name: intersect-sdk-common
+Version: 0.0.0a0
+Summary: Python SDK to interact with INTERSECT
+Keywords: intersect
+Author: Lance Drane, Marshall McDonnell, Seth Hitefield, Andrew Ayres, Gregory Cage, Jesse McGaha, Robert Smith, Gavin Wiggins, Michael Brim, Rick Archibald, Addi Malviya Thakur
+Author-email: Lance Drane <dranelt@ornl.gov>, Marshall McDonnell <mcdonnellmt@ornl.gov>, Seth Hitefield <hitefieldsd@ornl.gov>, Andrew Ayres <ayresaf@ornl.gov>, Gregory Cage <cagege@ornl.gov>, Jesse McGaha <mcgahajr@ornl.gov>, Robert Smith <smithrw@ornl.gov>, Gavin Wiggins <wigginsg@ornl.gov>, Michael Brim <brimmj@ornl.gov>, Rick Archibald <archibaldrk@ornl.gov>, Addi Malviya Thakur <malviyaa@ornl.gov>
+License: BSD-3-Clause
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: pydantic>=2.7.0
+Requires-Dist: paho-mqtt>=2.1.0,<3.0.0
+Requires-Dist: pika>=1.3.2,<2.0.0
+Requires-Dist: minio>=7.2.3
+Requires-Python: >=3.10, <4.0
+Description-Content-Type: text/markdown
+
+# INTERSECT common libraries
+
+This library is meant to provide common definitions across all INTERSECT services, both domain science microservices and INTERSECT core services.
+
+## IMPORTANT - READ THIS SECTION
+
+If you are developing a domain science application, do NOT import directly from this package. Use the [intersect-sdk](https://github.com/INTERSECT-SDK/python-sdk) package instead. You also don't need to import this package directly, the `intersect-sdk` package will automatically manage this version for you.
+
+## Version release policy
+
+Note that this package does _not_ follow semantic versioning for everything, as this is meant to be an internal package used by both the INTERSECT SDK and INTERSECT core services; it ONLY follows semantic versioning for the internal message structure.
+
+Prior to release 1.0.0, the policy is:
+
+- MINOR VERSION CHANGE - if the message structure changes (relevant to end users and the ecosystem)
+- PATCH VERSION CHANGE - any release, which may or may not be breaking (only relevant to the INTERSECT-SDK and INTERSECT core services)
+
+After release 1.0.0, the policy is expected to be:
+
+- MAJOR VERSION CHANGE - if the message structure changes (relevant to end users and the ecosystem)
+- MINOR VERSION CHANGE - breaking API changes (this is only relevant to the INTERSECT-SDK and INTERSECT core services)
+- PATCH VERSION CHANGE - backwards compatible changes
+
+The INTERSECT-SDK MUST follow semantic versioning, and MUST update its semantic version if this package's semantic version is updated.
+
+## Developing
+
+```bash
+uv venv .venv
+source .venv/bin/activate
+uv sync --locked --all-extras --all-groups
+```

intersect_sdk_common-0.0.0a0/README.md

@@ -0,0 +1,32 @@
+# INTERSECT common libraries
+
+This library is meant to provide common definitions across all INTERSECT services, both domain science microservices and INTERSECT core services.
+
+## IMPORTANT - READ THIS SECTION
+
+If you are developing a domain science application, do NOT import directly from this package. Use the [intersect-sdk](https://github.com/INTERSECT-SDK/python-sdk) package instead. You also don't need to import this package directly, the `intersect-sdk` package will automatically manage this version for you.
+
+## Version release policy
+
+Note that this package does _not_ follow semantic versioning for everything, as this is meant to be an internal package used by both the INTERSECT SDK and INTERSECT core services; it ONLY follows semantic versioning for the internal message structure.
+
+Prior to release 1.0.0, the policy is:
+
+- MINOR VERSION CHANGE - if the message structure changes (relevant to end users and the ecosystem)
+- PATCH VERSION CHANGE - any release, which may or may not be breaking (only relevant to the INTERSECT-SDK and INTERSECT core services)
+
+After release 1.0.0, the policy is expected to be:
+
+- MAJOR VERSION CHANGE - if the message structure changes (relevant to end users and the ecosystem)
+- MINOR VERSION CHANGE - breaking API changes (this is only relevant to the INTERSECT-SDK and INTERSECT core services)
+- PATCH VERSION CHANGE - backwards compatible changes
+
+The INTERSECT-SDK MUST follow semantic versioning, and MUST update its semantic version if this package's semantic version is updated.
+
+## Developing
+
+```bash
+uv venv .venv
+source .venv/bin/activate
+uv sync --locked --all-extras --all-groups
+```

intersect_sdk_common-0.0.0a0/pyproject.toml

@@ -0,0 +1,183 @@
+[project]
+name = "intersect-sdk-common"
+description = "Python SDK to interact with INTERSECT"
+authors = [
+    { name = "Lance Drane", email = "dranelt@ornl.gov" },
+    { name = "Marshall McDonnell", email = "mcdonnellmt@ornl.gov" },
+    { name = "Seth Hitefield", email = "hitefieldsd@ornl.gov" },
+    { name = "Andrew Ayres", email = "ayresaf@ornl.gov" },
+    { name = "Gregory Cage", email = "cagege@ornl.gov" },
+    { name = "Jesse McGaha", email = "mcgahajr@ornl.gov" },
+    { name = "Robert Smith", email = "smithrw@ornl.gov" },
+    { name = "Gavin Wiggins", email = "wigginsg@ornl.gov" },
+    { name = "Michael Brim", email = "brimmj@ornl.gov" },
+    { name = "Rick Archibald", email = "archibaldrk@ornl.gov" },
+    { name = "Addi Malviya Thakur", email = "malviyaa@ornl.gov" },
+]
+readme = "README.md"
+license = { text = "BSD-3-Clause" }
+requires-python = ">=3.10,<4.0"
+keywords = ["intersect"]
+version = "0.0.0alpha.0"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+dependencies = [
+    "pydantic>=2.7.0",
+    "paho-mqtt>=2.1.0,<3.0.0",
+    "pika>=1.3.2,<2.0.0",
+    "minio>=7.2.3",
+]
+
+[dependency-groups]
+lint = [
+    "pre-commit>=3.3.1",
+    "ruff==0.12.7",
+    "mypy>=1.10.0",
+    "codespell>=2.3.0",
+]
+test = ["pytest>=7.3.2", "pytest-cov>=4.1.0", "httpretty>=1.1.4"]
+
+[build-system]
+requires = ["uv_build>=0.10.2,<0.11.0"]
+build-backend = "uv_build"
+
+[tool.ruff]
+line-length = 100
+format = { quote-style = 'single' }
+
+[tool.ruff.lint]
+isort = { known-first-party = ['src'] }
+pydocstyle = { convention = 'google' }
+flake8-quotes = { inline-quotes = 'single', multiline-quotes = 'double' }
+mccabe = { max-complexity = 20 }
+pylint = { max-args = 10, max-branches = 20, max-returns = 15, max-statements = 75 }
+# pyflakes and the relevant pycodestyle rules are already configured
+extend-select = [
+    'C90',   # mccabe complexity
+    'I',     # isort
+    'N',     # pep8-naming
+    'D',     # pydocstyle
+    'UP',    # pyupgrade
+    'YTT',   # flake8-2020
+    'ANN',   # flake8-annotations
+    'ASYNC', # flake8-async
+    'S',     # flake8-bandit
+    'BLE',   # flake8-blind-except
+    'B',     # flake8-bugbear
+    'A',     # flake8-builtins
+    'COM',   # flake8-commas
+    'C4',    # flake8-comprehensions
+    'DTZ',   # flake8-datetimez
+    'T10',   # flake8-debugger
+    'EM',    # flake8-error-message
+    'FA',    # flake8-future-annotations
+    'ISC',   # flake8-implicit-string-concat
+    'ICN',   # flake8-import-conventions
+    'G',     # flake8-logging-format
+    'INP',   # flake8-no-pep420
+    'PIE',   # flake8-PIE
+    'T20',   # flake8-T20
+    'PYI',   # flake8-pyi
+    'PT',    # flake8-pytest-style
+    'Q',     # flake8-quotes
+    'RSE',   # flake8-raise
+    'RET',   # flake8-return
+    'SLF',   # flake8-self
+    'SLOT',  # flake8-slots
+    'SIM',   # flake8-simplify
+    'TC',   # flake8-type-checking
+    'ARG',   # flake8-unused-arguments
+    'PTH',   # flake8-use-pathlib
+    'PGH',   # pygrep-hooks
+    'PL',    # pylint
+    'TRY',   # tryceratops
+    'FLY',   # flynt
+    'RUF',   # RUFF additional rules
+    'INT',   # flake8-gettext
+]
+# If you're seeking to disable a rule, first consider whether the rule is overbearing, or if it should only be turned off for your usecase.
+ignore = [
+    'COM812',  # formatter, handled by Ruff format
+    'ISC001',  # formatter, handled by Ruff format
+    'SIM105',  # "with contextlib.suppress():" is slower than try-except-pass
+    'ANN401',  # allow explicit "Any" typing, use with care
+    'PLR2004', # allow "magic numbers"
+]
+
+[tool.ruff.lint.flake8-type-checking]
+runtime-evaluated-base-classes = ["pydantic.BaseModel"]
+runtime-evaluated-decorators = ["pydantic.dataclasses.dataclass","pydantic.validate_call"]
+
+[tool.ruff.lint.extend-per-file-ignores]
+'__init__.py' = [
+    'F401', # __init__.py commonly has unused imports
+    'TC004', # do lazy imports when importing from the base module
+]
+'docs/*' = [
+    'D',      # the documentation folder does not need documentation
+    'INP001', # docs are not a namespace package
+]
+'tests/*' = [
+    'S101',   # allow assert statements in tests
+    'S106',   # don't care about credentials in tests
+    'S311',   # don't care about cryptographic security in tests
+    'SLF001', # allow private member access in tests
+    'ANN',    # tests in general don't need types, unless they are runtime types.
+    'ARG',    # allow unused parameters in tests
+    'D',      # ignore documentation in tests
+    'RUF012', # permit "mutable" class attributes to not be annotated with typing.ClassVar (these shouldn't be mutated anyways...)
+ ]
+
+# see https://mypy.readthedocs.io/en/stable/config_file.html for a complete reference
+[tool.mypy]
+strict = true
+ignore_missing_imports = true       # don't require typing for library stubs if they don't exist
+disallow_untyped_decorators = false # this is needed for library decorator compatibility, i.e. "retrying"
+plugins = ["pydantic.mypy"]
+
+[tool.pydantic-mypy]
+init_forbid_extra = true
+init_typed = true
+warn_required_dynamic_aliases = true
+warn_untyped_fields = true
+
+[tool.pytest.ini_options]
+log_cli = true
+addopts = "-ra"
+
+[tool.coverage.report]
+omit = [
+    '*__init__*', # __init__ files should just re-export other classes and functions
+]
+exclude_also = [
+  'pragma: no-cover',              # standard
+  'if (typing\\.)?TYPE_CHECKING:', # type checking blocks are not executed in coverage, but we don't care
+  '@(abc\\.)?abstractmethod',      # don't try to cover abstract methods
+  "class .*\\bProtocol\\):",       # don't cover protocol classes (similar to abstract classes)
+  'raise NotImplementedError',     # it's not implemented so shouldn't be covered
+  'except.* ImportError',          # these are usually used to throw a "friendlier" error and are not really worth testing
+]
+
+
+#[project.scripts]
+#test-all = "pytest tests/ --cov=src/intersect_sdk_common/ --cov-fail-under=80 --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml"
+#test-all-debug = "pytest tests/ --cov=src/intersect_sdk_common/ --cov-fail-under=80 --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml -s"
+#test-unit = "pytest tests/unit --cov=src/intersect_sdk_common/"
+#test-e2e = "pytest tests/e2e --cov=src/intersect_sdk_common/"
+#lint-docs = "sphinx-build -W --keep-going docs docs/_build"
+#lint-format = "ruff format"
+#lint-ruff = "ruff check --fix"
+#lint-mypy = "mypy src/intersect_sdk_common/"
+#lint-spelling = "codespell -w docs examples src tests *.md -S docs/_build"
+
+[tool.codespell]
+# Ref: https://github.com/codespell-project/codespell#using-a-config-file
+skip = '.git*,*.lock,.venv,.*cache/*,./docs/_build/*'
+check-hidden = true
+# ignore-regex = ''
+# ignore-words-list = ''

intersect_sdk_common-0.0.0a0/src/intersect_sdk_common/__init__.py

@@ -0,0 +1,72 @@
+"""Common definitions shared across ALL INTERSECT services, be they domain science microservices or INTERSECT core services.
+
+You can use the base package as the public API, no need to import from submodules.
+
+IMPORTANT: If you are building a scientific microservice, do NOT import from this package. Instead, import from intersect-sdk, which is the public-facing library intended for you. Relevant definitions from this package will be re-exported there.
+"""
+
+from importlib import import_module
+from typing import TYPE_CHECKING
+
+# import everything eagerly for IDEs/LSPs
+if TYPE_CHECKING:
+    from .config import (
+        ControlPlaneConfig,
+        ControlProvider,
+        DataStoreConfig,
+        DataStoreConfigMap,
+        HierarchyConfig,
+    )
+    from .control_plane.control_plane_manager import ControlPlaneManager
+    from .control_plane.definitions import MessageCallback
+    from .core_definitions import IntersectDataHandler, IntersectMimeType
+    from .data_plane.data_plane_manager import DataPlaneManager
+    from .version import __version__, intersect_sdk_version_info, intersect_sdk_version_string
+
+__all__ = (
+    'ControlPlaneConfig',
+    'ControlPlaneManager',
+    'ControlProvider',
+    'DataPlaneManager',
+    'DataStoreConfig',
+    'DataStoreConfigMap',
+    'HierarchyConfig',
+    'IntersectDataHandler',
+    'IntersectMimeType',
+    'MessageCallback',
+    '__version__',
+    'intersect_sdk_version_info',
+    'intersect_sdk_version_string',
+)
+
+# PEP 562 stuff: do lazy imports for people who just want to import from the top-level module
+
+__lazy_imports = {
+    'ControlPlaneConfig': '.config',
+    'ControlProvider': '.config',
+    'DataStoreConfig': '.config',
+    'DataStoreConfigMap': '.config',
+    'HierarchyConfig': '.config',
+    'ControlPlaneManager': '.control_plane.control_plane_manager',
+    'MessageCallback': '.control_plane.definitions',
+    'IntersectDataHandler': '.core_definitions',
+    'IntersectMimeType': '.core_definitions',
+    'DataPlaneManager': '.data_plane.data_plane_manager',
+    '__version__': '.version',
+    'intersect_sdk_version_info': '.version',
+    'intersect_sdk_version_string': '.version',
+}
+
+
+def __getattr__(attr_name: str) -> object:
+    attr_module = __lazy_imports.get(attr_name)
+    if attr_module:
+        module = import_module(attr_module, package=__spec__.parent)
+        return getattr(module, attr_name)
+
+    msg = f'module {__name__!r} has no attribute {attr_name!r}'
+    raise AttributeError(msg)
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

intersect_sdk_common-0.0.0a0/src/intersect_sdk_common/config.py

@@ -0,0 +1,168 @@
+"""Configuration types shared across both Clients and Services."""
+
+from dataclasses import dataclass, field
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, PositiveInt
+
+from .core_definitions import IntersectDataHandler
+
+HIERARCHY_REGEX = r'^[a-z]((?!--)[a-z0-9-]){2,62}$'
+"""
+The hierarchy regex needs to be fairly restricted due to the number of different
+systems we want to be compatible with. The rules:
+
+- Only allow unreserved characters (alphanumeric and .-~_): https://datatracker.ietf.org/doc/html/rfc3986#section-2.3
+- Require lowercase letters to avoid incompatibilities with case-insensitive systems.
+- MinIO has been found to forbid _ and ~ characters
+- MinIO requires an alphanumeric character at the start of the string
+- No adjacent non-alphanumeric characters allowed
+- Range should be from 3-63 characters
+
+The following commit tracks several issues with MINIO: https://code.ornl.gov/intersect/additive-manufacturing/ros-intersect-adapter/-/commit/fa71b791be0ccf1a5884910b5be3b5239cf9896f
+"""
+
+ControlProvider = Literal['mqtt5.0', 'amqp0.9.1']
+"""The type of broker we connect to."""
+
+
+class HierarchyConfig(BaseModel):
+    """Configuration for registering this service in a system-of-system architecture."""
+
+    service: Annotated[str, Field(pattern=HIERARCHY_REGEX)]
+    """
+    The name of this application - should be unique within an INTERSECT system
+    """
+
+    subsystem: str | None = Field(default=None, pattern=HIERARCHY_REGEX)
+    """
+    An associated subsystem / service-grouping of the system (should be unique within an INTERSECT system)
+    """
+
+    system: Annotated[str, Field(pattern=HIERARCHY_REGEX)]
+    """
+    Name of the "system", could also be thought of as a "device" (should be unique within a facility)
+    """
+
+    facility: Annotated[str, Field(pattern=HIERARCHY_REGEX)]
+    """
+    Name of the facility (an ORNL institutional designation, i.e. 'neutrons') (NOT abbreviated, should be unique within an organization)
+    """
+
+    organization: Annotated[str, Field(pattern=HIERARCHY_REGEX)]
+    """
+    Name of the organization (i.e. 'ornl') (NOT abbreviated) (should be unique in an INTERSECT cluster)
+    """
+
+    def hierarchy_string(self, join_str: str = '') -> str:
+        """Get the full hierarchy string. This is mostly used internally, but if you're developing a client, it could potentially be helpful.
+
+        Params
+          join_str: String used to separate different hierarchy parts in the full string (default: empty string).
+
+        Returns:
+          Single string, which will contain all system-of-system parts. For optional parts not configured (i.e. - no subsystem), they will be represented by a "-" character.
+        """
+        if not self.subsystem:
+            return join_str.join([self.organization, self.facility, self.system, '-', self.service])
+        return join_str.join(
+            [
+                self.organization,
+                self.facility,
+                self.system,
+                self.subsystem,
+                self.service,
+            ]
+        )
+
+    # we need to use the Python regex engine instead of the Rust regex engine here, because Rust's does not support lookaheads
+    model_config = ConfigDict(regex_engine='python-re')
+
+
+@dataclass
+class ControlPlaneConfig:
+    """Configuration for interacting with a broker."""
+
+    protocol: ControlProvider
+    """
+    The protocol of the broker you'd like to use (i.e. AMQP, MQTT...)
+    """
+    # TODO - support more protocols and protocol versions as needed - see https://www.asyncapi.com/docs/reference/specification/v2.6.0#serverObject
+
+    username: Annotated[str, Field(min_length=1)]
+    """
+    Username credentials for broker connection.
+    """
+
+    password: Annotated[str, Field(min_length=1)]
+    """
+    Password credentials for broker connection.
+    """
+
+    host: Annotated[str, Field(min_length=1)] = '127.0.0.1'
+    """
+    Broker hostname (default: 127.0.0.1)
+    """
+
+    port: PositiveInt | None = None
+    """
+    Broker port. List of common ports:
+
+    - 1883 (MQTT)
+    - 4222 (NATS default port)
+    - 5222 (XMPP)
+    - 5223 (XMPP over TLS)
+    - 5671 (AMQP over TLS)
+    - 5672 (AMQP)
+    - 7400 (DDS Discovery)
+    - 7401 (DDS User traffic)
+    - 8883 (MQTT over TLS)
+    - 61613 (RabbitMQ STOMP - WARNING: ephemeral port)
+
+    NOTE: INTERSECT currently only supports AMQP and MQTT.
+    """
+
+
+@dataclass
+class DataStoreConfig:
+    """Configuration for interacting with a data store."""
+
+    username: Annotated[str, Field(min_length=1)]
+    """
+    Username credentials for data store connection.
+    """
+
+    password: Annotated[str, Field(min_length=1)]
+    """
+    Password credentials for data store connection.
+    """
+
+    host: Annotated[str, Field(min_length=1)] = '127.0.0.1'
+    """
+    Data store hostname (default: 127.0.0.1)
+    """
+
+    port: PositiveInt | None = None
+    """
+    Data store port
+    """
+
+
+@dataclass
+class DataStoreConfigMap:
+    """Configurations for any data stores the application should talk to."""
+
+    minio: list[DataStoreConfig] = field(default_factory=list)
+    """
+    minio configurations
+    """
+
+    def get_missing_data_store_types(self) -> set[IntersectDataHandler]:
+        """Return a set of IntersectDataHandlers which will not be permitted, due to a configuration type missing.
+
+        If all data configurations exist, returns an empty set
+        """
+        missing = set()
+        if not self.minio:
+            missing.add(IntersectDataHandler.MINIO)
+        return missing

intersect_sdk_common-0.0.0a0/src/intersect_sdk_common/constants.py

@@ -0,0 +1,28 @@
+"""These are miscellaneous constants used in INTERSECT which SDK users may obtain value from knowing about."""
+
+SYSTEM_OF_SYSTEM_REGEX = r'^[a-z0-9][-a-z0-9.]*[-a-z0-9]$'
+"""
+This is the regex used as a representation of a source/destination.
+This is only needed externally if you are building a client, services can ignore this.
+
+NOTE: for future compatibility reasons, we are NOT specifying the number of "parts" (separated by a '.') in this regex. All that matters is that you don't start or end with a period, or start with a hyphen.
+"""
+
+# see the internal schema file for full validation details
+CAPABILITY_REGEX = r'^[a-zA-Z0-9]\w*$'
+"""
+This is the regex used for representing capabilities and event keys. Capabilities should start with an alphanumeric character, and not be longer than 255 characters.
+
+This regex applies to namespacing local to a Service, so does not have to be unique across the ecosystem.
+"""
+
+MIME_TYPE_REGEX = r'\w+/[-+.\w]+'
+"""
+Regex used for validating Content Types.
+
+References can be found at:
+
+- https://www.iana.org/assignments/media-types/media-types.xhtml
+
+- https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
+"""

intersect_sdk_common-0.0.0a0/src/intersect_sdk_common/control_plane/__init__.py

@@ -0,0 +1 @@
+"""Logic associated with the control plane."""

intersect_sdk_common-0.0.0a0/src/intersect_sdk_common/control_plane/brokers/__init__.py

@@ -0,0 +1,4 @@
+"""Common logic for interacting with brokers on the protocol level.
+
+These classes should be kept internal to the common library.
+"""