plexus-python-pulse-common 1.1.77__py3-none-any.whl → 1.1.79__py3-none-any.whl

plexus/pulse/common/utils/flowutils.py

@@ -9,28 +9,62 @@ from collections.abc import Generator
 from typing import Self
 
 import pydantic as pdt
+import requests
 import sqlalchemy as sa
 import sqlalchemy.dialects.sqlite as sa_sqlite
 import sqlalchemy.orm as sa_orm
 from plexus.common.utils.dbutils import ConnectionMaker
 from plexus.common.utils.funcutils import Chainable
 from plexus.common.utils.funcutils import chainable, singleton
+from plexus.common.utils.jsonutils import JsonObject
+from plexus.common.utils.retry import retry
 from sqlmodel import Field
 
 from plexus.pulse.common.utils.ormutils import SQLiteUUID, SequenceModelMixinProtocol
 from plexus.pulse.common.utils.ormutils import clone_sequence_model_instance, make_base_model, make_sequence_model_mixin
 
 __all__ = [
+    "flow_workspace_root_path",
+    "current_flow_run_id",
+    "current_flow_workspace_root_path",
     "FlowStageEnum",
     "current_flow_stage",
-    "current_flow_run_id",
+    "stage_can_resolve_dataset_ref",
+    "make_dataset_ref_model_mixin",
     "DatasetRef",
     "DatasetRefTable",
-    "flow_context_file_path",
+    "current_flow_context_file_path",
     "FlowContext",
-    "context",
+    "flow_context",
+    "flow_api_url",
+    "resolve_dataset_ref",
+    "resolve_dataset_storage_location",
+    "resolve",
 ]
 
+flow_stage_env = "PULSE_FLOW_STAGE"
+flow_run_id_env = "PULSE_FLOW_RUN_ID"
+flow_workspace_root_dir_env = "PULSE_FLOW_WORKSPACE_ROOT_DIR"
+flow_api_url_env = "PULSE_FLOW_API_URL"
+
+
+def flow_workspace_root_path() -> pathlib.Path:
+    value = os.environ.get(flow_workspace_root_dir_env)
+    if value is None:
+        raise ValueError(f"Missing required environment variable '{flow_workspace_root_dir_env}'")
+    return pathlib.Path(value).absolute()
+
+
+def current_flow_run_id() -> str:
+    value = os.environ.get(flow_run_id_env)
+    if value is None:
+        raise ValueError(f"Missing required environment variable '{flow_run_id_env}'")
+    return value
+
+
+def current_flow_workspace_root_path() -> pathlib.Path:
+    return flow_workspace_root_path() / current_flow_run_id()
+
 
 class FlowStageEnum(enum.Enum):
     Preprocessing = "preprocessing"
@@ -40,20 +74,19 @@ class FlowStageEnum(enum.Enum):
 
 
 def current_flow_stage() -> FlowStageEnum:
-    stage_str = os.environ.get("PULSE_FLOW_STAGE")
-    if stage_str is None:
-        raise ValueError("Missing required environment variable 'PULSE_FLOW_STAGE'")
+    value = os.environ.get(flow_stage_env)
+    if value is None:
+        raise ValueError(f"Missing required environment variable '{flow_stage_env}'")
     for stage in list(FlowStageEnum):
-        if stage_str == stage.value:
+        if value == stage.value:
             return stage
-    raise ValueError(f"Illegal 'PULSE_FLOW_STAGE' value '{stage_str}'")
+    raise ValueError(f"Illegal '{flow_stage_env}' value '{value}'")
 
 
-def current_flow_run_id() -> str:
-    run_id_str = os.environ.get("PULSE_FLOW_RUN_ID")
-    if run_id_str is None:
-        raise ValueError("Missing required environment variable 'PULSE_FLOW_RUN_ID'")
-    return run_id_str
+def stage_can_resolve_dataset_ref(stage: FlowStageEnum | None = None) -> bool:
+    if stage is None:
+        stage = current_flow_stage()
+    return stage in {FlowStageEnum.Curation}
 
 
 def make_dataset_ref_model_mixin() -> type[pdt.BaseModel]:
@@ -62,10 +95,16 @@ def make_dataset_ref_model_mixin() -> type[pdt.BaseModel]:
             sa_column=sa.Column(SQLiteUUID(), nullable=False),
             description="UUID of the dataset",
         )
-        dataset_name: str | None = Field(
-            sa_column=sa.Column(sa_sqlite.VARCHAR(128), nullable=True),
+        dataset_name: str = Field(
+            sa_column=sa.Column(sa_sqlite.VARCHAR(128), nullable=False),
             description="Name of the dataset",
         )
+        dataset_props: JsonObject | None = Field(
+            sa_column=sa.Column(sa_sqlite.JSON),
+            default=None,
+            description="Additional properties stored as JSON",
+            schema_extra=dict(examples=[{"example_prop_key": "example_prop_value"}]),
+        )
 
     return ModelMixin
 
@@ -84,30 +123,16 @@ class DatasetRefTable(DatasetRef, make_sequence_model_mixin("sqlite"), table=Tru
 if typing.TYPE_CHECKING:
     class DatasetRef(BaseModel):
         dataset_uuid: sa_orm.Mapped[py_uuid.UUID] = ...
-        dataset_name: sa_orm.Mapped[str | None] = ...
+        dataset_name: sa_orm.Mapped[str] = ...
+        dataset_props: sa_orm.Mapped[JsonObject | None] = ...
 
 
     class DatasetRefTable(DatasetRef, SequenceModelMixinProtocol):
         pass
 
 
-def is_resolve_dataset_ref_stage(stage: FlowStageEnum | None) -> bool:
-    if stage is None:
-        stage = current_flow_stage()
-    return stage in {FlowStageEnum.Curation}
-
-
-def is_construct_dataset_ref_stage(stage: FlowStageEnum | None) -> bool:
-    if stage is None:
-        stage = current_flow_stage()
-    return stage in {FlowStageEnum.Training}
-
-
-def flow_context_file_path() -> pathlib.Path:
-    root_dir = os.environ.get("PULSE_FLOW_CONTEXT_ROOT_DIR")
-    if root_dir is None:
-        raise ValueError("Missing required environment variable 'PULSE_FLOW_CONTEXT_ROOT_DIR'")
-    return pathlib.Path(root_dir) / current_flow_run_id() / "flow_context.db"
+def current_flow_context_file_path() -> pathlib.Path:
+    return current_flow_workspace_root_path() / "flow_context.db"
 
 
 class FlowContext(object):
@@ -117,7 +142,7 @@ class FlowContext(object):
         file_path: str | os.PathLike[str] | None = None,
         fail_if_exists: bool = False,
     ):
-        self.file_path = pathlib.Path(file_path or flow_context_file_path())
+        self.file_path = pathlib.Path(file_path or current_flow_context_file_path())
         if fail_if_exists and self.file_path.exists():
             raise FileExistsError(f"flow context file '{self.file_path}' already exists")
         self.file_path.parent.mkdir(parents=True, exist_ok=True)
@@ -148,13 +173,32 @@ class FlowContext(object):
         with self.make_session() as session:
             return session.query(DatasetRefTable).all()
 
+    def query_dataset_ref(
+        self,
+        dataset_uuid: py_uuid.UUID | None = None,
+        dataset_name: str | None = None,
+    ) -> Chainable[Self, DatasetRefTable | None]:
+        if dataset_uuid is None and dataset_name is None:
+            raise ValueError("either 'dataset_uuid' or 'dataset_name' is required")
+
+        with self.make_session() as session:
+            query_stmt = session.query(DatasetRefTable)
+            if dataset_uuid is not None:
+                query_stmt = query_stmt.filter(DatasetRefTable.dataset_uuid == dataset_uuid)
+            if dataset_name is not None:
+                query_stmt = query_stmt.filter(DatasetRefTable.dataset_name == dataset_name)
+            db_dataset_ref = query_stmt.one_or_none()
+
+            return chainable(self, db_dataset_ref)
+
     def add_dataset_ref(
         self,
         dataset_uuid: py_uuid.UUID,
-        dataset_name: str | None = None,
+        dataset_name: str,
+        dataset_props: JsonObject | None = None,
     ) -> Chainable[Self, DatasetRefTable]:
         with self.make_session() as session:
-            dataset_ref = DatasetRef(dataset_uuid=dataset_uuid, dataset_name=dataset_name)
+            dataset_ref = DatasetRef(dataset_uuid=dataset_uuid, dataset_name=dataset_name, dataset_props=dataset_props)
             db_dataset_ref = clone_sequence_model_instance(DatasetRefTable, dataset_ref)
             session.add(db_dataset_ref)
             session.commit()
@@ -163,5 +207,85 @@ class FlowContext(object):
 
 
 @singleton
-def context() -> FlowContext:
+def flow_context() -> FlowContext:
     return FlowContext()
+
+
+def flow_api_url() -> str:
+    value = os.environ.get(flow_api_url_env)
+    if value is None:
+        raise ValueError(f"Missing required environment variable '{flow_api_url_env}'")
+    return value
+
+
+@retry(wait=10, retrials=3)
+def fetch_dataset_ref(
+    dataset_uuid: py_uuid.UUID | None = None,
+    dataset_name: str | None = None,
+) -> DatasetRef | None:
+    response = requests.get(f"{flow_api_url()}/pulse_demeter/dataset_public/query_dataset",
+                            headers={"accept": "application/json"},
+                            params={
+                                "dataset_uuid": str(dataset_uuid),
+                                "dataset_name": dataset_name,
+                            })
+    response.raise_for_status()
+
+    response_data = response.json()
+
+    try:
+        uuid = py_uuid.UUID(response_data["dataset"]["uuid"])
+        name = response_data["dataset"]["name"]
+        props = response_data["dataset"].get("props", None)
+        return DatasetRef(dataset_uuid=uuid, dataset_name=name, dataset_props=props)
+    except KeyError:
+        return None
+
+
+def resolve_dataset_ref(
+    dataset_uuid: py_uuid.UUID | None = None,
+    dataset_name: str | None = None,
+    *,
+    context: FlowContext | None = None,
+) -> DatasetRefTable:
+    if context is None:
+        context = flow_context()
+
+    if dataset_uuid is None and dataset_name is None:
+        raise ValueError("either 'dataset_uuid' or 'dataset_name' is required")
+
+    _, db_dataset_ref = context.query_dataset_ref(dataset_uuid=dataset_uuid, dataset_name=dataset_name)
+    if db_dataset_ref is None:
+        if not stage_can_resolve_dataset_ref():
+            # In some stages, the dataset ref should have been resolved and stored in the flow context.
+            # If not found, it's an error.
+            raise ValueError(f"cannot resolve dataset ref at flow stage '{current_flow_stage()}'")
+
+        dataset_ref = fetch_dataset_ref(dataset_uuid=dataset_uuid, dataset_name=dataset_name)
+        if dataset_ref is None:
+            raise ValueError(f"cannot found dataset with uuid '{str(dataset_uuid)}' or name '{dataset_name}'")
+
+        _, db_dataset_ref = context.add_dataset_ref(dataset_ref.dataset_uuid,
+                                                    dataset_ref.dataset_name,
+                                                    dataset_ref.dataset_props)
+
+    return db_dataset_ref
+
+
+def resolve_dataset_storage_location(
+    dataset_uuid: str | py_uuid.UUID | None = None,
+    dataset_name: str | None = None,
+    *,
+    context: FlowContext | None = None,
+) -> pathlib.Path:
+    if isinstance(dataset_uuid, str):
+        return resolve_dataset_storage_location(py_uuid.UUID(dataset_uuid), dataset_name, context=context)
+
+    db_dataset_ref = resolve_dataset_ref(dataset_uuid, dataset_name, context=context)
+    if "storage_location" in (db_dataset_ref.dataset_props or {}):
+        return pathlib.Path(db_dataset_ref.dataset_props["storage_location"]).absolute()
+
+    return current_flow_workspace_root_path() / "datasets" / str(db_dataset_ref.dataset_uuid)
+
+
+resolve = resolve_dataset_storage_location

plexus/pulse/common/utils/tagutils.py

@@ -26,7 +26,7 @@ from plexus.common.utils.iterutils import dicttree_add, dicttree_remove
 from plexus.common.utils.iterutils import dicttree_children, dicttree_lineage, dicttree_subtree
 from plexus.common.utils.jsonutils import JsonObject, JsonType
 from plexus.common.utils.randutils import randomizer
-from plexus.common.utils.strutils import is_blank
+from plexus.common.utils.strutils import is_blank, parse_bool_or
 from sqlmodel import Field
 
 from plexus.pulse.common.resources.tags import predefined_tagset_specs
@@ -47,6 +47,7 @@ __all__ = [
     "MutableTagset",
     "populate_tagset",
     "predefined_tagsets",
+    "seek_predefined_tagset_of",
     "render_tagset_markdown_readme",
     "make_tag_target_model_mixin",
     "make_tag_record_model_mixin",
@@ -356,6 +357,23 @@ def predefined_tagsets() -> dict[str, Tagset]:
     return tagsets
 
 
+def seek_predefined_tagset_of(
+    tag: str | Tag | BoundTag,
+    *,
+    tagset_namespace: str | None = None,
+    tagset_version: str | None = None,
+) -> Tagset | None:
+    if isinstance(tag, BoundTag):
+        versioned_namespace = versioned_identifier(tagset_namespace or tag.namespace, tagset_version or tag.version)
+    else:
+        versioned_namespace = versioned_identifier(tagset_namespace, tagset_version)
+
+    tagset = predefined_tagsets().get(versioned_namespace)
+    if tagset is None or tag not in tagset:
+        return None
+    return tagset
+
+
 def render_tagset_markdown_readme(tagset: Tagset) -> str:
     def render_desc(desc: str | RichDesc | None) -> str:
         if desc is None:
@@ -628,6 +646,16 @@ def tag_cache_file_path() -> pathlib.Path:
     return pathlib.Path.home() / ".local" / "plexus" / "tag_cache" / f"{randomizer().random_alphanumeric(7)}.db"
 
 
+tag_cache_strict_mode_env = "PULSE_TAG_CACHE_STRICT_MODE"
+
+
+def is_tag_cache_strict_mode() -> bool:
+    value = os.environ.get(tag_cache_strict_mode_env)
+    if value is None:
+        return False
+    return parse_bool_or(value) or False
+
+
 class TagCache(object):
     def __init__(
         self,
@@ -1089,6 +1117,13 @@ class TagCache(object):
                                instance, this parameter will be ignored and the version from the instance will be used.
         :return: Self instance for chaining along with the added ``TagRecordTable`` instance
         """
+        if is_tag_cache_strict_mode():
+            tagset = seek_predefined_tagset_of(tag, tagset_namespace=tagset_namespace, tagset_version=tagset_version)
+            if tagset is None:
+                raise ValueError(
+                    f"cannot find in the predefined tagset the specified tag '{tag}', with namespace '{tagset_namespace}' and version '{tagset_version}'"
+                )
+
         with self.make_session() as session:
             tag_record = TagRecord(
                 target_sqn=target_sqn,

{plexus_python_pulse_common-1.1.77.dist-info → plexus_python_pulse_common-1.1.79.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexus-python-pulse-common
-Version: 1.1.77
+Version: 1.1.79
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
@@ -16,12 +16,14 @@ Requires-Dist: jsonschema>=4.0
 Requires-Dist: lxml>=6.0
 Requires-Dist: markdown-it-py>=4.0
 Requires-Dist: numpy>=2.3
+Requires-Dist: packaging>=25.0
 Requires-Dist: pydantic>=2.8
 Requires-Dist: pygeodesy>=24.10
 Requires-Dist: pyparsing>=3.2
 Requires-Dist: pyproj>=3.6
 Requires-Dist: pyquaternion>=0.9
 Requires-Dist: pyyaml>=6.0
+Requires-Dist: requests>=2.0
 Requires-Dist: sqlalchemy>=2.0
 Requires-Dist: s3fs>=2025.10
 Requires-Dist: rich>=13.9

{plexus_python_pulse_common-1.1.77.dist-info → plexus_python_pulse_common-1.1.79.dist-info}/RECORD

@@ -14,7 +14,7 @@ plexus/pulse/common/utils/bagutils.py,sha256=5JGnMgR8Q9OciW5dE7_2U7n1FGVEXMZXv1S
 plexus/pulse/common/utils/config.py,sha256=YPtFK5QHTVsXwcFBGV0rlZad0GFtsgwysQNyKYjdzvk,2409
 plexus/pulse/common/utils/datautils.py,sha256=OWlnyFSa6UaPbSSG3HzCJHxdiHyfHAE-FUVHaLfNZoM,9335
 plexus/pulse/common/utils/dockerutils.py,sha256=7DF9yFGz8uHwpYk115xBWqOjs5etz0UE7yh66eDvBzs,5793
-plexus/pulse/common/utils/flowutils.py,sha256=VmbVcCVztH8DH47s_5JbdOZchsEc7hZbq1am95amJzs,5307
+plexus/pulse/common/utils/flowutils.py,sha256=MGu15KL9mw-LyGGLceVmY-c2FGer5PURnMZWFSJN--I,10182
 plexus/pulse/common/utils/gisutils.py,sha256=UX_IHPeAx9mbA9p_SsbPFwdsdp7bGeaQhWaCTCzvrBM,11466
 plexus/pulse/common/utils/jsonutils.py,sha256=zSbW94klN2jrkMdBECu4RNJX6zzNaSxauQ3FuihuB6E,3342
 plexus/pulse/common/utils/ormutils.py,sha256=JD-hKLEGkFWy7ojc3dC_90qtexe2UnLEvD90tEOHtCg,61897
@@ -22,9 +22,9 @@ plexus/pulse/common/utils/pathutils.py,sha256=S5xk2WEmxP8MiuSKIU3cAEtk4Nk5Lz7IqC
 plexus/pulse/common/utils/s3utils.py,sha256=RArl95rWoyBEFyWU9X0c8WgoQBjP_jOK3OkPoKxIMnA,36025
 plexus/pulse/common/utils/sqlutils.py,sha256=D6kTBjhO5YlNRt3uFlPt6z3uH61m9ajEzPYmsI6NoFc,231
 plexus/pulse/common/utils/strutils.py,sha256=PkmMHCcHN9vALCy_8fJTf00Dg7I2N4QTCXdBmB3t7rk,16440
-plexus/pulse/common/utils/tagutils.py,sha256=smM79dyRCiswKTTcI-At7FLxDH54VN4Id3aFHbDxzYo,61696
+plexus/pulse/common/utils/tagutils.py,sha256=-7cZWVpmtI7R8bB-SBtHPw4WyYJKp8AptH_Rfc-5sk4,62938
 plexus/pulse/common/utils/testutils.py,sha256=hKPRvoN4mg8QGQJH121l59a0uo8jN5rntGQAVYplGBw,6166
-plexus_python_pulse_common-1.1.77.dist-info/METADATA,sha256=Aei_-DjiA9AuGRzaDCZ5Vp3R8dJxcjXJSKFZ7SZwRHQ,1495
-plexus_python_pulse_common-1.1.77.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-plexus_python_pulse_common-1.1.77.dist-info/top_level.txt,sha256=ug_g7CVwaMQuas5UzAXbHUrQvKGCn8ezc6ZNvvRlJOE,7
-plexus_python_pulse_common-1.1.77.dist-info/RECORD,,
+plexus_python_pulse_common-1.1.79.dist-info/METADATA,sha256=yWNshrrquuxOYcZVF9KyXRMxqqTUZVOviSYLaapI_Co,1555
+plexus_python_pulse_common-1.1.79.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+plexus_python_pulse_common-1.1.79.dist-info/top_level.txt,sha256=ug_g7CVwaMQuas5UzAXbHUrQvKGCn8ezc6ZNvvRlJOE,7
+plexus_python_pulse_common-1.1.79.dist-info/RECORD,,