plexus-python-pulse-demeter 1.1.126__py3-none-any.whl

plexus/pulse/demeter/app/models/registry/tagging.py

@@ -0,0 +1,478 @@
+import datetime
+import typing
+import uuid as py_uuid
+from typing import Self
+
+import jsonschema
+import pydantic as pdt
+import sqlalchemy as sa
+import sqlalchemy.dialects.postgresql as sa_pg
+import sqlalchemy.orm as sa_orm
+from plexus.common.utils.dtutils import dt_to_ts_us
+from plexus.common.utils.jsonutils import JsonObject
+from plexus.pulse.common.utils.datautils import (
+    validate_colon_tag,
+    validate_dt_timezone,
+    validate_semver,
+    validate_slash_tag,
+    validate_snake_case,
+    validate_vehicle_name,
+)
+from plexus.pulse.common.utils.jsonutils import json_datetime_encoder
+from plexus.pulse.common.utils.ormutils import ChangingModelMixinProtocol, SequenceModelMixinProtocol
+from plexus.pulse.common.utils.ormutils import (
+    changing_model_mixin,
+    make_changing_model_mixin,
+    make_sequence_model_mixin,
+)
+from sqlmodel import Field, SQLModel
+
+from plexus.pulse.demeter.app.models.registry import RegistryBase
+
+__all__ = [
+    "DataTag",
+    "DataTagTarget",
+    "DataTagRecord",
+    "DataTagClip",
+    "DataTagTable",
+    "DataTagTargetTable",
+    "DataTagRecordTable",
+    "DataTagClipTable",
+]
+
+
+class DataTag(RegistryBase):
+    namespace: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(64), nullable=False),
+        description="Namespace of the tagset that this tag belongs to",
+        schema_extra=dict(examples=["namespace_name"]),
+    )
+    version: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(32), nullable=False),
+        description="Version of the tagset that this tag belongs to",
+        schema_extra=dict(examples=["1.2.3-alpha+post0"]),
+    )
+    name: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(256), nullable=False),
+        description="Name of the tag",
+        schema_extra=dict(examples=["tag_name:sub_tag_name"]),
+    )
+    parent_name: str | None = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(256)),
+        default=None,
+        description="Name of the parent tag that this tag extends",
+        schema_extra=dict(examples=["tag_name:sub_tag_name"]),
+    )
+    props_schema: JsonObject | None = Field(
+        sa_column=sa.Column(sa_pg.JSONB),
+        default=None,
+        description="JSON schema defining the properties format for this tag",
+        schema_extra=dict(
+            examples=[
+                {
+                    "type": "object",
+                    "required": ["weather", "obstacle"],
+                    "properties": {
+                        "weather": {"type": "string", "enum": ["sunny", "rainy", "foggy"]},
+                        "obstacle": {
+                            "type": "string",
+                            "enum": ["car", "truck", "bus", "motorcycle", "pedestrian", "bicycle"],
+                        },
+                    },
+                },
+            ]
+        ),
+    )
+
+    @pdt.field_validator("namespace", mode="after")
+    @classmethod
+    def validate_namespace(cls, v: str) -> str:
+        validate_snake_case(v)
+        return v
+
+    @pdt.field_validator("version", mode="after")
+    @classmethod
+    def validate_version(cls, v: str) -> str:
+        validate_semver(v)
+        return v
+
+    @pdt.field_validator("name", mode="after")
+    @classmethod
+    def validate_name(cls, v: str) -> str | None:
+        validate_colon_tag(v)
+        return v
+
+    @pdt.field_validator("parent_name", mode="after")
+    @classmethod
+    def validate_parent_name(cls, v: str | None) -> str | None:
+        if v is not None:
+            validate_colon_tag(v)
+        return v
+
+    @pdt.field_validator("props_schema", mode="after")
+    @classmethod
+    def validate_props_schema(cls, v: JsonObject | None) -> JsonObject | None:
+        if v is not None:
+            try:
+                jsonschema.Draft7Validator.check_schema(v)
+            except Exception as e:
+                raise ValueError("not a valid JSON schema") from e
+        return v
+
+
+class DataTagTarget(RegistryBase):
+    uuid: py_uuid.UUID = Field(
+        sa_column=sa.Column(sa_pg.UUID(as_uuid=True), nullable=False),
+        default_factory=py_uuid.uuid4,
+        description="UUID of the tag target",
+        schema_extra=dict(examples=["abcd1234-5678-90ab-cdef-1234567890ab"]),
+    )
+    tagger_name: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(128), nullable=False),
+        description="Name of the tagger that generates the tag records for the target",
+        schema_extra=dict(examples=["tagger_name"]),
+    )
+    tagger_version: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(32), nullable=False),
+        description="Version of the tagger that generates the tag records for the target",
+        schema_extra=dict(examples=["1.2.3-alpha+post0"]),
+    )
+    vehicle_name: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(64), nullable=False),
+        description="Name of the tagger's applicability vehicle",
+        schema_extra=dict(examples=["vehicle_name"]),
+    )
+    begin_dt: datetime.datetime = Field(
+        sa_column=sa.Column(sa_pg.TIMESTAMP(timezone=True), nullable=False),
+        description="Beginning of the tagger's applicability time range",
+        schema_extra=dict(examples=["2024-01-01T00:00:00.000000+00:00"]),
+    )
+    end_dt: datetime.datetime = Field(
+        sa_column=sa.Column(sa_pg.TIMESTAMP(timezone=True), nullable=False),
+        description="End of the tagger's applicability time range",
+        schema_extra=dict(examples=["2024-12-31T23:59:59.999999+00:00"]),
+    )
+    props: JsonObject | None = Field(
+        sa_column=sa.Column(sa_pg.JSONB),
+        default=None,
+        description="Properties of the tag record stored as JSON",
+        schema_extra=dict(examples=[{"example_prop_key": "example_prop_value"}]),
+    )
+
+    @pdt.field_validator("tagger_name", mode="after")
+    @classmethod
+    def validate_tagger_name(cls, v: str) -> str:
+        validate_slash_tag(v)
+        return v
+
+    @pdt.field_validator("tagger_version", mode="after")
+    @classmethod
+    def validate_tagger_version(cls, v: str) -> str:
+        validate_semver(v)
+        return v
+
+    @pdt.field_validator("vehicle_name", mode="after")
+    @classmethod
+    def validate_vehicle_name(cls, v: str) -> str:
+        validate_vehicle_name(v)
+        return v
+
+    @pdt.field_validator("begin_dt", mode="after")
+    @classmethod
+    def validate_begin_dt(cls, v: datetime.datetime) -> datetime.datetime:
+        validate_dt_timezone(v)
+        return v
+
+    @pdt.field_validator("end_dt", mode="after")
+    @classmethod
+    def validate_end_dt(cls, v: datetime.datetime) -> datetime.datetime:
+        validate_dt_timezone(v)
+        return v
+
+    @pdt.model_validator(mode="after")
+    def validate_begin_dt_end_dt(self) -> Self:
+        if self.begin_dt > self.end_dt:
+            raise ValueError(f"begin_dt '{self.begin_dt}' is greater than end_dt '{self.end_dt}'")
+        return self
+
+    @pdt.field_serializer("begin_dt", mode="plain")
+    def serialize_begin_dt(self, v: datetime.datetime) -> str:
+        return json_datetime_encoder(v)
+
+    @pdt.field_serializer("end_dt", mode="plain")
+    def serialize_end_dt(self, v: datetime.datetime) -> str:
+        return json_datetime_encoder(v)
+
+
+class DataTagRecord(RegistryBase):
+    target_uuid: py_uuid.UUID = Field(
+        sa_column=sa.Column(sa_pg.UUID(as_uuid=True), nullable=False),
+        description="UUID of the tag target that this record is associated with",
+        schema_extra=dict(examples=["abcd1234-5678-90ab-cdef-1234567890ab"]),
+    )
+    vehicle_name: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(64), nullable=False),
+        description="Name of the vehicle associated with this tag record",
+        schema_extra=dict(examples=["vehicle_name"]),
+    )
+    begin_dt: datetime.datetime = Field(
+        sa_column=sa.Column(sa_pg.TIMESTAMP(timezone=True), nullable=False),
+        description="Beginning of the tag record time range",
+        schema_extra=dict(examples=["2024-01-01T00:00:00.000000+00:00"]),
+    )
+    end_dt: datetime.datetime = Field(
+        sa_column=sa.Column(sa_pg.TIMESTAMP(timezone=True), nullable=False),
+        description="End of the tag record time range",
+        schema_extra=dict(examples=["2024-01-01T01:00:00.000000+00:00"]),
+    )
+    tagset_namespace: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(64), nullable=False),
+        description="Namespace of the tagset that the tag in this record belongs to",
+        schema_extra=dict(examples=["namespace_name"]),
+    )
+    tagset_version: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(32), nullable=False),
+        description="Version of the tagset that the tag in this record belongs to",
+        schema_extra=dict(examples=["1.2.3-alpha+post0"]),
+    )
+    tag: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(256), nullable=False),
+        description="Tag name",
+        schema_extra=dict(examples=["tag_name:sub_tag_name"]),
+    )
+    features: JsonObject | None = Field(
+        sa_column=sa.Column(sa_pg.JSONB),
+        default=None,
+        description="Extracted features associated with the tag record, stored as JSON",
+        schema_extra=dict(examples=[{"example_feature_key": "example_feature_value"}]),
+    )
+    props: JsonObject | None = Field(
+        sa_column=sa.Column(sa_pg.JSONB),
+        default=None,
+        description="Properties of the tag record stored as JSON",
+        schema_extra=dict(examples=[{"example_prop_key": "example_prop_value"}]),
+    )
+
+    @pdt.field_validator("vehicle_name", mode="after")
+    @classmethod
+    def validate_vehicle_name(cls, v: str) -> str:
+        validate_vehicle_name(v)
+        return v
+
+    @pdt.field_validator("begin_dt", mode="after")
+    @classmethod
+    def validate_begin_dt(cls, v: datetime.datetime) -> datetime.datetime:
+        validate_dt_timezone(v)
+        return v
+
+    @pdt.field_validator("end_dt", mode="after")
+    @classmethod
+    def validate_end_dt(cls, v: datetime.datetime) -> datetime.datetime:
+        validate_dt_timezone(v)
+        return v
+
+    @pdt.model_validator(mode="after")
+    def validate_begin_dt_end_dt(self) -> Self:
+        if self.begin_dt > self.end_dt:
+            raise ValueError(f"begin_dt '{self.begin_dt}' is greater than end_dt '{self.end_dt}'")
+        return self
+
+    @pdt.field_validator("tag", mode="after")
+    @classmethod
+    def validate_tag(cls, v: str) -> str:
+        validate_colon_tag(v)
+        return v
+
+    @pdt.field_serializer("begin_dt", mode="plain")
+    def serialize_begin_dt(self, v: datetime.datetime) -> str:
+        return json_datetime_encoder(v)
+
+    @pdt.field_serializer("end_dt", mode="plain")
+    def serialize_end_dt(self, v: datetime.datetime) -> str:
+        return json_datetime_encoder(v)
+
+
+class DataTagClip(RegistryBase):
+    source_sqn: int = Field(
+        sa_column=sa.Column(sa_pg.BIGINT, nullable=False),
+        description="Source sequence number for the tag clip, used for ordering and lineage tracking",
+    )
+    vehicle_name: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(64), nullable=False),
+        description="Name of the vehicle associated with this tag record",
+        schema_extra=dict(examples=["vehicle_name"]),
+    )
+    clip_dt: datetime.datetime = Field(
+        sa_column=sa.Column(sa_pg.TIMESTAMP(timezone=True), nullable=False),
+        description="Timestamp representing the time of the tag clip",
+        schema_extra=dict(examples=["2024-01-01T00:00:00.000000+00:00"]),
+    )
+    clip_duration_us: int = Field(
+        sa_column=sa.Column(sa_pg.BIGINT, nullable=False),
+        description="Duration of the tag clip in microseconds",
+        schema_extra=dict(examples=[1_000_000]),
+    )
+    begin_offset_us: int = Field(
+        sa_column=sa.Column(sa_pg.BIGINT, nullable=False),
+        description="Offset in microseconds from the beginning of the tag clip to the beginning of the tag record time range",
+        schema_extra=dict(examples=[0]),
+    )
+    end_offset_us: int = Field(
+        sa_column=sa.Column(sa_pg.BIGINT, nullable=False),
+        description="Offset in microseconds from the beginning of the tag clip to the end of the tag record time range",
+        schema_extra=dict(examples=[1_000_000]),
+    )
+    tagset_namespace: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(64), nullable=False),
+        description="Namespace of the tagset that the tag in this record belongs to",
+        schema_extra=dict(examples=["namespace_name"]),
+    )
+    tagset_version: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(32), nullable=False),
+        description="Version of the tagset that the tag in this record belongs to",
+        schema_extra=dict(examples=["1.2.3-alpha+post0"]),
+    )
+    tag: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(256), nullable=False),
+        description="Tag name",
+        schema_extra=dict(examples=["tag_name:sub_tag_name"]),
+    )
+
+    @pdt.field_validator("vehicle_name", mode="after")
+    @classmethod
+    def validate_vehicle_name(cls, v: str) -> str:
+        validate_vehicle_name(v)
+        return v
+
+    @pdt.field_validator("clip_dt", mode="after")
+    @classmethod
+    def validate_clip_dt(cls, v: datetime.datetime) -> datetime.datetime:
+        validate_dt_timezone(v)
+        return v
+
+    @pdt.field_validator("clip_duration_us", mode="after")
+    @classmethod
+    def validate_clip_duration_us(cls, v: int) -> int:
+        if v <= 0:
+            raise ValueError(f"clip_duration_us '{v}' must be a positive integer")
+        return v
+
+    @pdt.field_validator("begin_offset_us", mode="after")
+    @classmethod
+    def validate_begin_offset_us(cls, v: int) -> int:
+        if v < 0:
+            raise ValueError(f"begin_offset_us '{v}' must be a non-negative integer")
+        return v
+
+    @pdt.field_validator("end_offset_us", mode="after")
+    @classmethod
+    def validate_end_offset_us(cls, v: int) -> int:
+        if v < 0:
+            raise ValueError(f"end_offset_us '{v}' must be a non-negative integer")
+        return v
+
+    @pdt.model_validator(mode="after")
+    def validate_clip_dt_clip_duration_us(self) -> Self:
+        if dt_to_ts_us(self.clip_dt) % self.clip_duration_us > 0:
+            raise ValueError(
+                f"clip_dt '{self.clip_dt}' must be aligned with clip_duration_us '{self.clip_duration_us}'")
+        return self
+
+    @pdt.model_validator(mode="after")
+    def validate_clip_duration_us_begin_offset_us_end_offset_us(self) -> Self:
+        if self.begin_offset_us > self.end_offset_us:
+            raise ValueError(
+                f"begin_offset_us '{self.begin_offset_us}' is greater than end_offset_us '{self.end_offset_us}'")
+        if self.end_offset_us > self.clip_duration_us:
+            raise ValueError(
+                f"end_offset_us '{self.end_offset_us}' is greater than clip_duration_us '{self.clip_duration_us}'")
+        return self
+
+    @pdt.field_validator("tag", mode="after")
+    @classmethod
+    def validate_tag(cls, v: str) -> str:
+        validate_colon_tag(v)
+        return v
+
+    @pdt.field_serializer("clip_dt", mode="plain")
+    def serialize_clip_dt(self, v: datetime.datetime) -> str:
+        return json_datetime_encoder(v)
+
+
+class DataTagTable(DataTag, make_changing_model_mixin(), table=True):
+    __tablename__ = "data_tag"
+    __table_args__ = (
+        changing_model_mixin.make_index_created_at("ix_data_tag_created_at"),
+        sa.Index("ix_data_tag_namespace_version_name", "namespace", "version", "name"),
+    )
+
+
+class DataTagTargetTable(DataTagTarget, make_changing_model_mixin(), table=True):
+    __tablename__ = "data_tag_target"
+    __table_args__ = (
+        changing_model_mixin.make_index_created_at("ix_data_tag_target_created_at"),
+        sa.Index("ix_data_tag_target_uuid", "uuid", unique=True),
+        sa.Index("ix_data_tag_target_tagger_name_tagger_version", "tagger_name", "tagger_version"),
+        sa.Index("ix_data_tag_target_vehicle_name_begin_dt_end_dt", "vehicle_name", "begin_dt", "end_dt"),
+    )
+
+
+class DataTagRecordTable(DataTagRecord, make_changing_model_mixin(), table=True):
+    __tablename__ = "data_tag_record"
+    __table_args__ = (
+        changing_model_mixin.make_index_created_at("ix_data_tag_record_created_at"),
+        sa.Index("ix_data_tag_record_vehicle_name_begin_dt_end_dt", "vehicle_name", "begin_dt", "end_dt"),
+    )
+
+
+class DataTagClipTable(DataTagClip, make_sequence_model_mixin(), table=True):
+    __tablename__ = "data_tag_clip"
+    __table_args__ = (
+        sa.Index("ix_data_tag_clip_source_sqn", "source_sqn"),
+        sa.Index("ix_data_tag_clip_vehicle_name_clip_dt", "vehicle_name", "clip_dt"),
+    )
+
+
+if typing.TYPE_CHECKING:
+    RegistryBase = SQLModel
+
+
+    class DataTagTable(RegistryBase, ChangingModelMixinProtocol):
+        namespace: sa_orm.Mapped[str] = ...
+        version: sa_orm.Mapped[str] = ...
+        name: sa_orm.Mapped[str] = ...
+        parent_name: sa_orm.Mapped[str | None] = ...
+        props_schema: sa_orm.Mapped[JsonObject | None] = ...
+
+
+    class DataTagTargetTable(RegistryBase, ChangingModelMixinProtocol):
+        uuid: sa_orm.Mapped[py_uuid.UUID] = ...
+        tagger_name: sa_orm.Mapped[str] = ...
+        tagger_version: sa_orm.Mapped[str] = ...
+        vehicle_name: sa_orm.Mapped[str] = ...
+        begin_dt: sa_orm.Mapped[datetime.datetime] = ...
+        end_dt: sa_orm.Mapped[datetime.datetime] = ...
+
+
+    class DataTagRecordTable(RegistryBase, ChangingModelMixinProtocol):
+        target_uuid: sa_orm.Mapped[py_uuid.UUID] = ...
+        vehicle_name: sa_orm.Mapped[str] = ...
+        begin_dt: sa_orm.Mapped[datetime.datetime] = ...
+        end_dt: sa_orm.Mapped[datetime.datetime] = ...
+        tagset_namespace: sa_orm.Mapped[str] = ...
+        tagset_version: sa_orm.Mapped[str] = ...
+        tag: sa_orm.Mapped[str] = ...
+        features: sa_orm.Mapped[JsonObject | None] = ...
+        props: sa_orm.Mapped[JsonObject | None] = ...
+
+
+    class DataTagClipTable(RegistryBase, SequenceModelMixinProtocol):
+        source_sqn: sa_orm.Mapped[int] = ...
+        vehicle_name: sa_orm.Mapped[str] = ...
+        clip_dt: sa_orm.Mapped[datetime.datetime] = ...
+        clip_duration_us: sa_orm.Mapped[int] = ...
+        begin_offset_us: sa_orm.Mapped[int] = ...
+        end_offset_us: sa_orm.Mapped[int] = ...
+        tagset_namespace: sa_orm.Mapped[str] = ...
+        tagset_version: sa_orm.Mapped[str] = ...
+        tag: sa_orm.Mapped[str] = ...

plexus/pulse/demeter/app/models/registry/user.py

@@ -0,0 +1,79 @@
+import enum
+import typing
+
+import pydantic as pdt
+import sqlalchemy as sa
+import sqlalchemy.dialects.postgresql as sa_pg
+import sqlalchemy.orm as sa_orm
+from plexus.pulse.common.utils.datautils import validate_user_name
+from plexus.pulse.common.utils.ormutils import RevisionModelMixinProtocol
+from plexus.pulse.common.utils.ormutils import make_revision_model_mixin, revision_model_mixin
+from sqlmodel import Field, SQLModel
+
+from plexus.pulse.demeter.app.models.registry import RegistryBase
+
+__all__ = [
+    "UserRole",
+    "User",
+    "UserTable",
+]
+
+
+class UserRole(enum.Enum):
+    Admin = "admin"
+    Contributor = "contributor"
+    Reviewer = "reviewer"
+    Viewer = "viewer"
+
+
+class User(RegistryBase):
+    name: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(64), nullable=False),
+        description="Name of the user",
+        schema_extra=dict(examples=["user.name"]),
+    )
+    sso_sid: int | None = Field(
+        sa_column=sa.Column(sa_pg.BIGINT),
+        default=None,
+        description="SSO SID of the user",
+        schema_extra=dict(examples=[1234567890]),
+    )
+    role: str = Field(
+        sa_column=sa.Column(sa_pg.VARCHAR(32), nullable=False),
+        description="Role of the account",
+        schema_extra=dict(examples=["contributor"]),
+    )
+
+    @pdt.field_validator("name", mode="after")
+    @classmethod
+    def validate_name(cls, v: str) -> str:
+        validate_user_name(v)
+        return v
+
+    @pdt.field_validator("role", mode="after")
+    @classmethod
+    def validate_role(cls, v: str) -> str:
+        if not any(v == item.value for item in UserRole):
+            raise ValueError(f"invalid user role '{v}'")
+        return v
+
+
+class UserTable(User, make_revision_model_mixin(), table=True):
+    __tablename__ = "user"
+    __table_args__ = (
+        revision_model_mixin.make_index_created_at_updated_at_expired_at("ix_user_created_at_updated_at_expired_at"),
+        revision_model_mixin.make_unique_index_record_sqn_revision("ix_u_user_record_sqn_revision"),
+        revision_model_mixin.make_active_unique_index_record_sqn("ix_ua_user_record_sqn"),
+        revision_model_mixin.make_active_unique_index_for("ix_ua_user_name", "name"),
+        revision_model_mixin.make_active_unique_index_for("ix_ua_user_sso_sid", "sso_sid"),
+    )
+
+
+if typing.TYPE_CHECKING:
+    RegistryBase = SQLModel
+
+
+    class UserTable(RegistryBase, RevisionModelMixinProtocol):
+        name: sa_orm.Mapped[str] = ...
+        sso_sid: sa_orm.Mapped[int | None] = ...
+        role: sa_orm.Mapped[str] = ...