plexus-python-pulse-common 1.1.76__tar.gz → 1.1.78__tar.gz

{plexus_python_pulse_common-1.1.76 → plexus_python_pulse_common-1.1.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexus-python-pulse-common
-Version: 1.1.76
+Version: 1.1.78
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
@@ -22,6 +22,7 @@ Requires-Dist: pyparsing>=3.2
 Requires-Dist: pyproj>=3.6
 Requires-Dist: pyquaternion>=0.9
 Requires-Dist: pyyaml>=6.0
+Requires-Dist: requests>=2.0
 Requires-Dist: sqlalchemy>=2.0
 Requires-Dist: s3fs>=2025.10
 Requires-Dist: rich>=13.9

{plexus_python_pulse_common-1.1.76 → plexus_python_pulse_common-1.1.78}/pyproject.toml

@@ -24,6 +24,7 @@ dev = [
     "pyproj>=3.6",
     "pyquaternion>=0.9",
     "pyyaml>=6.0",
+    "requests>=2.0",
     "sqlalchemy>=2.0",
     "s3fs>=2025.10",
     "rich>=13.9",
@@ -71,6 +72,7 @@ dependencies = [
     "pyproj>=3.6",
     "pyquaternion>=0.9",
     "pyyaml>=6.0",
+    "requests>=2.0",
     "sqlalchemy>=2.0",
     "s3fs>=2025.10",
     "rich>=13.9",

plexus_python_pulse_common-1.1.78/src/plexus/pulse/common/utils/flowutils.py

@@ -0,0 +1,291 @@
+import contextlib
+import enum
+import os
+import pathlib
+import threading
+import typing
+import uuid as py_uuid
+from collections.abc import Generator
+from typing import Self
+
+import pydantic as pdt
+import requests
+import sqlalchemy as sa
+import sqlalchemy.dialects.sqlite as sa_sqlite
+import sqlalchemy.orm as sa_orm
+from plexus.common.utils.dbutils import ConnectionMaker
+from plexus.common.utils.funcutils import Chainable
+from plexus.common.utils.funcutils import chainable, singleton
+from plexus.common.utils.jsonutils import JsonObject
+from plexus.common.utils.retry import retry
+from sqlmodel import Field
+
+from plexus.pulse.common.utils.ormutils import SQLiteUUID, SequenceModelMixinProtocol
+from plexus.pulse.common.utils.ormutils import clone_sequence_model_instance, make_base_model, make_sequence_model_mixin
+
+__all__ = [
+    "flow_workspace_root_path",
+    "current_flow_run_id",
+    "current_flow_workspace_root_path",
+    "FlowStageEnum",
+    "current_flow_stage",
+    "stage_can_resolve_dataset_ref",
+    "make_dataset_ref_model_mixin",
+    "DatasetRef",
+    "DatasetRefTable",
+    "current_flow_context_file_path",
+    "FlowContext",
+    "flow_context",
+    "flow_api_url",
+    "resolve_dataset_ref",
+    "resolve_dataset_storage_location",
+    "resolve",
+]
+
+flow_stage_env = "PULSE_FLOW_STAGE"
+flow_run_id_env = "PULSE_FLOW_RUN_ID"
+flow_workspace_root_dir_env = "PULSE_FLOW_WORKSPACE_ROOT_DIR"
+flow_api_url_env = "PULSE_FLOW_API_URL"
+
+
+def flow_workspace_root_path() -> pathlib.Path:
+    value = os.environ.get(flow_workspace_root_dir_env)
+    if value is None:
+        raise ValueError(f"Missing required environment variable '{flow_workspace_root_dir_env}'")
+    return pathlib.Path(value).absolute()
+
+
+def current_flow_run_id() -> str:
+    value = os.environ.get(flow_run_id_env)
+    if value is None:
+        raise ValueError(f"Missing required environment variable '{flow_run_id_env}'")
+    return value
+
+
+def current_flow_workspace_root_path() -> pathlib.Path:
+    return flow_workspace_root_path() / current_flow_run_id()
+
+
+class FlowStageEnum(enum.Enum):
+    Preprocessing = "preprocessing"
+    Annotation = "annotation"
+    Curation = "curation"
+    Training = "training"
+
+
+def current_flow_stage() -> FlowStageEnum:
+    value = os.environ.get(flow_stage_env)
+    if value is None:
+        raise ValueError(f"Missing required environment variable '{flow_stage_env}'")
+    for stage in list(FlowStageEnum):
+        if value == stage.value:
+            return stage
+    raise ValueError(f"Illegal '{flow_stage_env}' value '{value}'")
+
+
+def stage_can_resolve_dataset_ref(stage: FlowStageEnum | None = None) -> bool:
+    if stage is None:
+        stage = current_flow_stage()
+    return stage in {FlowStageEnum.Curation}
+
+
+def make_dataset_ref_model_mixin() -> type[pdt.BaseModel]:
+    class ModelMixin(pdt.BaseModel):
+        dataset_uuid: py_uuid.UUID = Field(
+            sa_column=sa.Column(SQLiteUUID(), nullable=False),
+            description="UUID of the dataset",
+        )
+        dataset_name: str = Field(
+            sa_column=sa.Column(sa_sqlite.VARCHAR(128), nullable=False),
+            description="Name of the dataset",
+        )
+        dataset_props: JsonObject | None = Field(
+            sa_column=sa.Column(sa_sqlite.JSON),
+            default=None,
+            description="Additional properties stored as JSON",
+            schema_extra=dict(examples=[{"example_prop_key": "example_prop_value"}]),
+        )
+
+    return ModelMixin
+
+
+BaseModel = make_base_model()
+
+
+class DatasetRef(BaseModel, make_dataset_ref_model_mixin()):
+    pass
+
+
+class DatasetRefTable(DatasetRef, make_sequence_model_mixin("sqlite"), table=True):
+    __tablename__ = "dataset_ref"
+
+
+if typing.TYPE_CHECKING:
+    class DatasetRef(BaseModel):
+        dataset_uuid: sa_orm.Mapped[py_uuid.UUID] = ...
+        dataset_name: sa_orm.Mapped[str] = ...
+        dataset_props: sa_orm.Mapped[JsonObject | None] = ...
+
+
+    class DatasetRefTable(DatasetRef, SequenceModelMixinProtocol):
+        pass
+
+
+def current_flow_context_file_path() -> pathlib.Path:
+    return current_flow_workspace_root_path() / "flow_context.db"
+
+
+class FlowContext(object):
+    def __init__(
+        self,
+        *,
+        file_path: str | os.PathLike[str] | None = None,
+        fail_if_exists: bool = False,
+    ):
+        self.file_path = pathlib.Path(file_path or current_flow_context_file_path())
+        if fail_if_exists and self.file_path.exists():
+            raise FileExistsError(f"flow context file '{self.file_path}' already exists")
+        self.file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        self.conn_maker = ConnectionMaker.from_url(f"sqlite:///{self.file_path.absolute()}",
+                                                   engine_opts=dict(connect_args={"check_same_thread": False}))
+
+        self.thread_lock = threading.Lock()
+
+        BaseModel.metadata.create_all(self.conn_maker.engine)
+
+    @contextlib.contextmanager
+    def make_session(self) -> Generator[sa_orm.Session, None, None]:
+        with self.thread_lock:
+            with self.conn_maker.make_session(expire_on_commit=False) as session:
+                try:
+                    yield session
+                    session.commit()
+                except Exception:
+                    session.rollback()
+                    raise
+
+    def count_dataset_ref(self) -> int:
+        with self.make_session() as session:
+            return session.query(DatasetRefTable).count()
+
+    def list_dataset_refs(self) -> list[DatasetRefTable]:
+        with self.make_session() as session:
+            return session.query(DatasetRefTable).all()
+
+    def query_dataset_ref(
+        self,
+        dataset_uuid: py_uuid.UUID | None = None,
+        dataset_name: str | None = None,
+    ) -> Chainable[Self, DatasetRefTable | None]:
+        if dataset_uuid is None and dataset_name is None:
+            raise ValueError("either 'dataset_uuid' or 'dataset_name' is required")
+
+        with self.make_session() as session:
+            query_stmt = session.query(DatasetRefTable)
+            if dataset_uuid is not None:
+                query_stmt = query_stmt.filter(DatasetRefTable.dataset_uuid == dataset_uuid)
+            if dataset_name is not None:
+                query_stmt = query_stmt.filter(DatasetRefTable.dataset_name == dataset_name)
+            db_dataset_ref = query_stmt.one_or_none()
+
+            return chainable(self, db_dataset_ref)
+
+    def add_dataset_ref(
+        self,
+        dataset_uuid: py_uuid.UUID,
+        dataset_name: str,
+        dataset_props: JsonObject | None = None,
+    ) -> Chainable[Self, DatasetRefTable]:
+        with self.make_session() as session:
+            dataset_ref = DatasetRef(dataset_uuid=dataset_uuid, dataset_name=dataset_name, dataset_props=dataset_props)
+            db_dataset_ref = clone_sequence_model_instance(DatasetRefTable, dataset_ref)
+            session.add(db_dataset_ref)
+            session.commit()
+
+            return chainable(self, db_dataset_ref)
+
+
+@singleton
+def flow_context() -> FlowContext:
+    return FlowContext()
+
+
+def flow_api_url() -> str:
+    value = os.environ.get(flow_api_url_env)
+    if value is None:
+        raise ValueError(f"Missing required environment variable '{flow_api_url_env}'")
+    return value
+
+
+@retry(wait=10, retrials=3)
+def fetch_dataset_ref(
+    dataset_uuid: py_uuid.UUID | None = None,
+    dataset_name: str | None = None,
+) -> DatasetRef | None:
+    response = requests.get(f"{flow_api_url()}/pulse_demeter/dataset_public/query_dataset",
+                            headers={"accept": "application/json"},
+                            params={
+                                "dataset_uuid": str(dataset_uuid),
+                                "dataset_name": dataset_name,
+                            })
+    response.raise_for_status()
+
+    response_data = response.json()
+
+    try:
+        uuid = py_uuid.UUID(response_data["dataset"]["uuid"])
+        name = response_data["dataset"]["name"]
+        props = response_data["dataset"].get("props", None)
+        return DatasetRef(dataset_uuid=uuid, dataset_name=name, dataset_props=props)
+    except KeyError:
+        return None
+
+
+def resolve_dataset_ref(
+    dataset_uuid: py_uuid.UUID | None = None,
+    dataset_name: str | None = None,
+    *,
+    context: FlowContext | None = None,
+) -> DatasetRefTable:
+    if context is None:
+        context = flow_context()
+
+    if dataset_uuid is None and dataset_name is None:
+        raise ValueError("either 'dataset_uuid' or 'dataset_name' is required")
+
+    _, db_dataset_ref = context.query_dataset_ref(dataset_uuid=dataset_uuid, dataset_name=dataset_name)
+    if db_dataset_ref is None:
+        if not stage_can_resolve_dataset_ref():
+            # In some stages, the dataset ref should have been resolved and stored in the flow context.
+            # If not found, it's an error.
+            raise ValueError(f"cannot resolve dataset ref at flow stage '{current_flow_stage()}'")
+
+        dataset_ref = fetch_dataset_ref(dataset_uuid=dataset_uuid, dataset_name=dataset_name)
+        if dataset_ref is None:
+            raise ValueError(f"cannot found dataset with uuid '{str(dataset_uuid)}' or name '{dataset_name}'")
+
+        _, db_dataset_ref = context.add_dataset_ref(dataset_ref.dataset_uuid,
+                                                    dataset_ref.dataset_name,
+                                                    dataset_ref.dataset_props)
+
+    return db_dataset_ref
+
+
+def resolve_dataset_storage_location(
+    dataset_uuid: str | py_uuid.UUID | None = None,
+    dataset_name: str | None = None,
+    *,
+    context: FlowContext | None = None,
+) -> pathlib.Path:
+    if isinstance(dataset_uuid, str):
+        return resolve_dataset_storage_location(py_uuid.UUID(dataset_uuid), dataset_name, context=context)
+
+    db_dataset_ref = resolve_dataset_ref(dataset_uuid, dataset_name, context=context)
+    if "storage_location" in (db_dataset_ref.dataset_props or {}):
+        return pathlib.Path(db_dataset_ref.dataset_props["storage_location"]).absolute()
+
+    return current_flow_workspace_root_path() / "datasets" / str(db_dataset_ref.dataset_uuid)
+
+
+resolve = resolve_dataset_storage_location

{plexus_python_pulse_common-1.1.76 → plexus_python_pulse_common-1.1.78}/src/plexus/pulse/common/utils/ormutils.py

@@ -1,4 +1,5 @@
 import datetime
+import uuid as py_uuid
 from collections.abc import Iterable, Sequence
 from typing import Protocol, Self
 
@@ -28,8 +29,10 @@ __all__ = [
     "ChangingModelMixinProtocol",
     "SnapshotModelMixinProtocol",
     "RevisionModelMixinProtocol",
+    "SQLiteUUID",
     "SQLiteDateTime",
     "model_sqn_type",
+    "model_uuid_type",
     "model_datetime_tz_type",
     "model_revision_type",
     "SequenceModelMixin",
@@ -395,11 +398,29 @@ class RevisionModelMixinProtocol(SequenceModelMixinProtocol):
         ...
 
 
+class SQLiteUUID(sa.TypeDecorator):
+    """
+    Custom SQLAlchemy type decorator for handling UUIDs in SQLite.
+    """
+    impl = sa.CHAR(36)
+    cache_ok = True
+
+    def process_bind_param(self, value: py_uuid.UUID | None, dialect) -> str | None:
+        if value is None:
+            return None
+        return str(value)
+
+    def process_result_value(self, value: str | None, dialect) -> py_uuid.UUID | None:
+        if value is None:
+            return None
+        return py_uuid.UUID(value)
+
+
 class SQLiteDateTime(sa.TypeDecorator):
     """
     Custom SQLAlchemy type decorator for handling timezone-aware datetimes in SQLite.
     """
-    impl = sa.DateTime
+    impl = sa.DATETIME
     cache_ok = True
 
     def process_bind_param(self, value: datetime.datetime | None, dialect) -> datetime.datetime | None:
@@ -433,6 +454,22 @@ def model_sqn_type(dialect: str | None = None) -> sa.types.TypeEngine[int]:
     raise ValueError(f"unsupported database dialect '{dialect}'")
 
 
+def model_uuid_type(dialect: str | None = None) -> sa.types.TypeEngine[py_uuid.UUID]:
+    """
+    Returns the appropriate SQLAlchemy column type for a UUID based on the specified database dialect.
+
+    :param dialect: The database dialect to determine the column type for. If None, defaults to PostgreSQL.
+    :return: The SQLAlchemy column type to use for the UUID field.
+    """
+    if dialect is None:
+        dialect = dialects.postgresql
+    if dialect == dialects.postgresql:
+        return sa_pg.UUID(as_uuid=True)
+    if dialect == dialects.sqlite:
+        return SQLiteUUID
+    raise ValueError(f"unsupported database dialect '{dialect}'")
+
+
 def model_datetime_tz_type(dialect: str | None = None) -> sa.types.TypeEngine[datetime.datetime]:
     """
     Returns the appropriate SQLAlchemy column type for a timezone-aware datetime field based on the specified database

{plexus_python_pulse_common-1.1.76 → plexus_python_pulse_common-1.1.78}/src/plexus/pulse/common/utils/tagutils.py

@@ -26,7 +26,7 @@ from plexus.common.utils.iterutils import dicttree_add, dicttree_remove
 from plexus.common.utils.iterutils import dicttree_children, dicttree_lineage, dicttree_subtree
 from plexus.common.utils.jsonutils import JsonObject, JsonType
 from plexus.common.utils.randutils import randomizer
-from plexus.common.utils.strutils import is_blank
+from plexus.common.utils.strutils import is_blank, parse_bool_or
 from sqlmodel import Field
 
 from plexus.pulse.common.resources.tags import predefined_tagset_specs
@@ -47,6 +47,7 @@ __all__ = [
     "MutableTagset",
     "populate_tagset",
     "predefined_tagsets",
+    "seek_predefined_tagset_of",
     "render_tagset_markdown_readme",
     "make_tag_target_model_mixin",
     "make_tag_record_model_mixin",
@@ -356,6 +357,23 @@ def predefined_tagsets() -> dict[str, Tagset]:
     return tagsets
 
 
+def seek_predefined_tagset_of(
+    tag: str | Tag | BoundTag,
+    *,
+    tagset_namespace: str | None = None,
+    tagset_version: str | None = None,
+) -> Tagset | None:
+    if isinstance(tag, BoundTag):
+        versioned_namespace = versioned_identifier(tagset_namespace or tag.namespace, tagset_version or tag.version)
+    else:
+        versioned_namespace = versioned_identifier(tagset_namespace, tagset_version)
+
+    tagset = predefined_tagsets().get(versioned_namespace)
+    if tagset is None or tag not in tagset:
+        return None
+    return tagset
+
+
 def render_tagset_markdown_readme(tagset: Tagset) -> str:
     def render_desc(desc: str | RichDesc | None) -> str:
         if desc is None:
@@ -628,6 +646,16 @@ def tag_cache_file_path() -> pathlib.Path:
     return pathlib.Path.home() / ".local" / "plexus" / "tag_cache" / f"{randomizer().random_alphanumeric(7)}.db"
 
 
+tag_cache_strict_mode_env = "PULSE_TAG_CACHE_STRICT_MODE"
+
+
+def is_tag_cache_strict_mode() -> bool:
+    value = os.environ.get(tag_cache_strict_mode_env)
+    if value is None:
+        return False
+    return parse_bool_or(value) or False
+
+
 class TagCache(object):
     def __init__(
         self,
@@ -1089,6 +1117,13 @@ class TagCache(object):
                                instance, this parameter will be ignored and the version from the instance will be used.
         :return: Self instance for chaining along with the added ``TagRecordTable`` instance
         """
+        if is_tag_cache_strict_mode():
+            tagset = seek_predefined_tagset_of(tag, tagset_namespace=tagset_namespace, tagset_version=tagset_version)
+            if tagset is None:
+                raise ValueError(
+                    f"cannot find in the predefined tagset the specified tag '{tag}', with namespace '{tagset_namespace}' and version '{tagset_version}'"
+                )
+
         with self.make_session() as session:
             tag_record = TagRecord(
                 target_sqn=target_sqn,

{plexus_python_pulse_common-1.1.76 → plexus_python_pulse_common-1.1.78}/src/plexus_python_pulse_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexus-python-pulse-common
-Version: 1.1.76
+Version: 1.1.78
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
@@ -22,6 +22,7 @@ Requires-Dist: pyparsing>=3.2
 Requires-Dist: pyproj>=3.6
 Requires-Dist: pyquaternion>=0.9
 Requires-Dist: pyyaml>=6.0
+Requires-Dist: requests>=2.0
 Requires-Dist: sqlalchemy>=2.0
 Requires-Dist: s3fs>=2025.10
 Requires-Dist: rich>=13.9

{plexus_python_pulse_common-1.1.76 → plexus_python_pulse_common-1.1.78}/src/plexus_python_pulse_common.egg-info/SOURCES.txt

@@ -54,6 +54,7 @@ src/plexus/pulse/common/utils/bagutils.py
 src/plexus/pulse/common/utils/config.py
 src/plexus/pulse/common/utils/datautils.py
 src/plexus/pulse/common/utils/dockerutils.py
+src/plexus/pulse/common/utils/flowutils.py
 src/plexus/pulse/common/utils/gisutils.py
 src/plexus/pulse/common/utils/jsonutils.py
 src/plexus/pulse/common/utils/ormutils.py
@@ -80,6 +81,7 @@ test/plexus_tests/pulse/common/utils/__init__.py
 test/plexus_tests/pulse/common/utils/bagutils_test.py
 test/plexus_tests/pulse/common/utils/datautils_test.py
 test/plexus_tests/pulse/common/utils/dockerutils_test.py
+test/plexus_tests/pulse/common/utils/flowutils_test.py
 test/plexus_tests/pulse/common/utils/gisutils_test.py
 test/plexus_tests/pulse/common/utils/jsonutils_test.py
 test/plexus_tests/pulse/common/utils/ormutils_test.py

{plexus_python_pulse_common-1.1.76 → plexus_python_pulse_common-1.1.78}/src/plexus_python_pulse_common.egg-info/requires.txt

@@ -14,6 +14,7 @@ pyparsing>=3.2
 pyproj>=3.6
 pyquaternion>=0.9
 pyyaml>=6.0
+requests>=2.0
 sqlalchemy>=2.0
 s3fs>=2025.10
 rich>=13.9

plexus_python_pulse_common-1.1.78/test/plexus_tests/pulse/common/utils/flowutils_test.py

@@ -0,0 +1,48 @@
+import os
+import tempfile
+import unittest
+import uuid as pyuuid
+from unittest.mock import patch
+
+import ddt
+
+from plexus.pulse.common.utils.flowutils import FlowContext
+
+
+@ddt.ddt
+class FlowUtilsTest(unittest.TestCase):
+
+    def test_flow_context(self):
+        with tempfile.TemporaryDirectory(delete=True) as temp_dir:
+            with patch.dict(os.environ,
+                            {
+                                "PULSE_FLOW_WORKSPACE_ROOT_DIR": temp_dir,
+                                "PULSE_FLOW_RUN_ID": "01234567-89ab-cdef-0123-456789abcdef",
+                            },
+                            clear=True):
+                context = (
+                    FlowContext()
+                    .add_dataset_ref(pyuuid.UUID("01234567-89ab-cdef-0123-456789abcdef"), "one_dataset")
+                    .add_dataset_ref(pyuuid.UUID("89abcdef-0123-4567-89ab-cdef01234567"), "another_dataset")
+                    .add_dataset_ref(pyuuid.UUID("00000000-0000-0000-0000-000000000000"), "zero_dataset")
+                )
+
+                self.assertEqual(context.count_dataset_ref(), 3)
+
+                one_dataset, another_dataset, zero_dataset = context.list_dataset_refs()
+
+                self.assertEqual(one_dataset.dataset_uuid, pyuuid.UUID("01234567-89ab-cdef-0123-456789abcdef"))
+                self.assertEqual(another_dataset.dataset_uuid, pyuuid.UUID("89abcdef-0123-4567-89ab-cdef01234567"))
+                self.assertEqual(zero_dataset.dataset_uuid, pyuuid.UUID("00000000-0000-0000-0000-000000000000"))
+
+            # This is the same flow context file as above, but we are creating a new `FlowContext` instance
+            # via explicit file path which is deducted from the mocked environment variables
+            context = FlowContext(file_path=f"{temp_dir}/01234567-89ab-cdef-0123-456789abcdef/flow_context.db")
+
+            self.assertEqual(context.count_dataset_ref(), 3)
+
+            one_dataset, another_dataset, zero_dataset = context.list_dataset_refs()
+
+            self.assertEqual(one_dataset.dataset_uuid, pyuuid.UUID("01234567-89ab-cdef-0123-456789abcdef"))
+            self.assertEqual(another_dataset.dataset_uuid, pyuuid.UUID("89abcdef-0123-4567-89ab-cdef01234567"))
+            self.assertEqual(zero_dataset.dataset_uuid, pyuuid.UUID("00000000-0000-0000-0000-000000000000"))