plexus-python-pulse-demeter 1.1.126__py3-none-any.whl

plexus/pulse/demeter/app/api/tagging_endpoints.py

@@ -0,0 +1,323 @@
+import dataclasses
+import datetime
+import tempfile
+import uuid as py_uuid
+from collections.abc import Callable
+
+import pydantic as pdt
+import pyparsing as pp
+import sqlalchemy as sa
+import sqlalchemy.orm as sa_orm
+from fastapi import APIRouter, Depends, File, UploadFile
+from fastapi.params import Query
+from plexus.common.utils.dtutils import dt_utc_now, td_to_us
+from plexus.common.utils.funcutils import singleton
+from plexus.pulse.common.utils.apiutils import managed_db_session
+from plexus.pulse.common.utils.ormutils import (
+    db_create_changing_model,
+    db_create_changing_models,
+    db_create_sequence_models,
+)
+from plexus.pulse.common.utils.tagutils import TagCache
+from plexus.pulse.common.utils.tagutils import populate_clip_ranges, predefined_tagsets, standard_clip_duration_us
+from plexus.pulse.common.utils.tagutils import versioned_identifier
+
+from plexus.pulse.demeter.app.core.database import make_session
+from plexus.pulse.demeter.app.models import Pagination
+from plexus.pulse.demeter.app.models.registry.tagging import (
+    DataTagClip,
+    DataTagClipTable,
+    DataTagRecord,
+    DataTagRecordTable,
+    DataTagTarget,
+    DataTagTargetTable,
+)
+
+router = APIRouter(prefix="/pulse_demeter/tagging", tags=["tagging"])
+
+
+@singleton
+def tag_query_interpreter() -> Callable[[str], sa.ColumnElement[bool]]:
+    # AST node classes
+    @dataclasses.dataclass(frozen=True)
+    class TagQueryExpr:
+        tag_name: str
+        features_query: str | None = None
+
+    @dataclasses.dataclass(frozen=True)
+    class NotExpr:
+        operand: "Expr"
+
+    @dataclasses.dataclass(frozen=True)
+    class AndExpr:
+        left: "Expr"
+        right: "Expr"
+
+    @dataclasses.dataclass(frozen=True)
+    class OrExpr:
+        left: "Expr"
+        right: "Expr"
+
+    Expr = TagQueryExpr | NotExpr | AndExpr | OrExpr
+
+    # Parse actions
+    def make_tag_query_expr(tokens):
+        if len(tokens) == 1:
+            tag_name, = tokens
+            return TagQueryExpr(tag_name=tag_name, features_query=None)
+        elif len(tokens) == 2:
+            tag_name, features_query = tokens
+            return TagQueryExpr(tag_name=tag_name, features_query=features_query)
+        else:
+            raise ValueError(f"invalid tag query expression: {tokens}")
+
+    def make_not(tokens):
+        if len(tokens) == 1:
+            operand, *_ = tokens
+            return NotExpr(operand=operand)
+        raise ValueError(f"invalid NOT expression: {tokens}")
+
+    def make_and(tokens):
+        left, *operands = tokens[0][::2]
+        if len(operands) == 0:
+            raise ValueError(f"invalid AND expression: {tokens}")
+        for right in operands:
+            left = AndExpr(left=left, right=right)
+        return left
+
+    def make_or(tokens):
+        left, *operands = tokens[0][::2]
+        if len(operands) == 0:
+            raise ValueError(f"invalid OR expression: {tokens}")
+        for right in operands:
+            left = OrExpr(left=left, right=right)
+        return left
+
+    # Grammar
+    ident_token = pp.Regex(r"[a-zA-Z0-9]+(_[a-zA-Z0-9]+)*")
+    colon_token = pp.Literal(":")
+
+    tag_name_expr = (ident_token + (pp.Suppress(colon_token) + ident_token)[...])
+    tag_name_expr.set_parse_action(lambda tokens: ":".join(tokens))
+
+    features_query_expr = pp.nested_expr("{", "}", )
+    features_query_expr.set_parse_action(lambda tokens: "".join(map(str, tokens[0])))
+
+    tag_query_expr = (tag_name_expr("tag_name") + pp.Optional(features_query_expr("features_query")))
+    tag_query_expr.set_parse_action(make_tag_query_expr)
+
+    expr = pp.Forward()
+    expr <<= pp.infix_notation(
+        tag_query_expr,
+        [
+            (pp.Keyword("not"), 1, pp.OpAssoc.RIGHT, make_not),
+            (pp.Keyword("and"), 2, pp.OpAssoc.LEFT, make_and),
+            (pp.Keyword("or"), 2, pp.OpAssoc.LEFT, make_or),
+        ],
+    )
+
+    def ast_to_sa_filter(ast: Expr) -> sa.ColumnElement[bool]:
+        """
+        Converts the AST into an SQLAlchemy filter.
+        """
+        if isinstance(ast, TagQueryExpr):
+            if ast.features_query is None:
+                return DataTagRecordTable.tag == ast.tag_name
+            return sa.and_(
+                DataTagRecordTable.tag == ast.tag_name,
+                sa.text(f"jsonb_path_exists({DataTagRecordTable.__table__.c.features.name}, '{ast.features_query}')"),
+            )
+
+        if isinstance(ast, NotExpr):
+            return sa.not_(ast_to_sa_filter(ast.operand))
+
+        if isinstance(ast, AndExpr):
+            left_filter = ast_to_sa_filter(ast.left)
+            right_filter = ast_to_sa_filter(ast.right)
+            return sa.and_(left_filter, right_filter)
+
+        if isinstance(ast, OrExpr):
+            left_filter = ast_to_sa_filter(ast.left)
+            right_filter = ast_to_sa_filter(ast.right)
+            return sa.or_(left_filter, right_filter)
+
+        raise ValueError(f"unknown AST node type: {type(ast)}")
+
+    def converter(query: str) -> sa.ColumnElement[bool]:
+        ast, *_ = expr.parse_string(query, parse_all=True)
+        return ast_to_sa_filter(ast)
+
+    return converter
+
+
+class TagTargetsAndPagination(pdt.BaseModel):
+    targets: list[DataTagTargetTable]
+    pagination: Pagination
+
+
+@router.get("/targets", response_model=TagTargetsAndPagination)
+def list_targets(
+    tagger_name: str | None = None,
+    tagger_version: str | None = None,
+    vehicle_name: str | None = None,
+    begin_dt: datetime.datetime | None = None,
+    end_dt: datetime.datetime | None = None,
+    skip: int = Query(0),
+    limit: int = Query(1000),
+    db: sa_orm.Session = Depends(make_session),
+) -> TagTargetsAndPagination:
+    with managed_db_session(db):
+        query_stmt = db.query(DataTagTargetTable)
+        if begin_dt is not None:
+            query_stmt = query_stmt.filter(DataTagTargetTable.end_dt >= begin_dt)
+        if end_dt is not None:
+            query_stmt = query_stmt.filter(DataTagTargetTable.begin_dt <= end_dt)
+        if tagger_name is not None:
+            query_stmt = query_stmt.filter(DataTagTargetTable.tagger_name == tagger_name)
+        if tagger_version is not None:
+            query_stmt = query_stmt.filter(DataTagTargetTable.tagger_version == tagger_version)
+        if vehicle_name is not None:
+            query_stmt = query_stmt.filter(DataTagTargetTable.vehicle_name == vehicle_name)
+
+        query_stmt = query_stmt.order_by(DataTagTargetTable.sqn.desc())
+
+        total = query_stmt.count()
+        if total == 0:
+            return TagTargetsAndPagination(targets=[], pagination=Pagination(total=0, skip=skip, limit=limit))
+
+        db_targets = query_stmt.offset(skip).limit(limit).all()
+
+        return TagTargetsAndPagination(targets=db_targets, pagination=Pagination(total=total, skip=skip, limit=limit))
+
+
+class TagRecordsAndPagination(pdt.BaseModel):
+    records: list[DataTagRecordTable]
+    pagination: Pagination
+
+
+@router.get("/records", response_model=TagRecordsAndPagination)
+def list_records(
+    begin_dt: datetime.datetime | None = None,
+    end_dt: datetime.datetime | None = None,
+    tagset_namespace: str | None = None,
+    tagset_version: str | None = None,
+    query: str | None = Query(None, description="Tag query string (e.g. 'tag1 and tag2 {features_query} or not tag3')"),
+    tagger_name: str | None = Query(None),
+    tagger_version: str | None = Query(None),
+    vehicle_name: str | None = Query(None),
+    skip: int = Query(0),
+    limit: int = Query(1000),
+    db: sa_orm.Session = Depends(make_session),
+) -> TagRecordsAndPagination:
+    with managed_db_session(db):
+        query_stmt = db.query(DataTagRecordTable)
+        if begin_dt is not None:
+            query_stmt = query_stmt.filter(DataTagRecordTable.end_dt >= begin_dt)
+        if end_dt is not None:
+            query_stmt = query_stmt.filter(DataTagRecordTable.begin_dt <= end_dt)
+        if tagset_namespace is not None:
+            query_stmt = query_stmt.filter(DataTagRecordTable.tagset_namespace == tagset_namespace)
+        if tagset_version is not None:
+            query_stmt = query_stmt.filter(DataTagRecordTable.tagset_version == tagset_version)
+        if query is not None:
+            try:
+                interpreter = tag_query_interpreter()
+                query_filter = interpreter(query)
+                query_stmt = query_stmt.filter(query_filter)
+            except pp.ParseException as e:
+                raise ValueError(f"failed to parse tag query: {e}")
+        if tagger_name is not None or tagger_version is not None:
+            query_stmt = query_stmt.join(DataTagTargetTable, DataTagRecordTable.target_uuid == DataTagTargetTable.uuid)
+            if tagger_name is not None:
+                query_stmt = query_stmt.filter(DataTagTargetTable.tagger_name == tagger_name)
+            if tagger_version is not None:
+                query_stmt = query_stmt.filter(DataTagTargetTable.tagger_version == tagger_version)
+        if vehicle_name is not None:
+            query_stmt = query_stmt.filter(DataTagRecordTable.vehicle_name == vehicle_name)
+
+        query_stmt = query_stmt.order_by(DataTagRecordTable.sqn.desc())
+
+        total = query_stmt.count()
+        if total == 0:
+            return TagRecordsAndPagination(records=[], pagination=Pagination(total=0, skip=skip, limit=limit))
+
+        db_records = query_stmt.offset(skip).limit(limit).all()
+
+        return TagRecordsAndPagination(records=db_records, pagination=Pagination(total=total, skip=skip, limit=limit))
+
+
+class TagCacheSummary(pdt.BaseModel):
+    targets_count: int
+    records_count: int
+
+
+@router.post("/upload_tag_cache/", response_model=TagCacheSummary)
+def upload_tag_cache(
+    tag_cache_file: UploadFile = File(...),
+    created_at: datetime.datetime = Query(default_factory=dt_utc_now),
+    db: sa_orm.Session = Depends(make_session),
+) -> TagCacheSummary:
+    with managed_db_session(db), tempfile.NamedTemporaryFile() as temp_db_file:
+        temp_db_file.write(tag_cache_file.file.read())
+        temp_db_file.flush()
+
+        tag_cache = TagCache(file_path=temp_db_file.name)
+
+        for record in tag_cache.iter_records():
+            versioned_namespace = versioned_identifier(record.tagset_namespace, record.tagset_version)
+            if versioned_namespace not in predefined_tagsets():
+                raise ValueError(f"Tagset '{versioned_namespace}' is not predefined")
+
+        targets_count = 0
+        records_count = 0
+
+        for target in tag_cache.query_targets():
+            tag_target = DataTagTarget(
+                uuid=py_uuid.uuid4(),
+                tagger_name=target.tagger_name,
+                tagger_version=target.tagger_version,
+                vehicle_name=target.vehicle_name,
+                begin_dt=target.begin_dt,
+                end_dt=target.end_dt,
+            )
+
+            tag_records = [
+                DataTagRecord(
+                    target_uuid=tag_target.uuid,
+                    vehicle_name=tag_target.vehicle_name,
+                    begin_dt=record.begin_dt,
+                    end_dt=record.end_dt,
+                    tagset_namespace=record.tagset_namespace,
+                    tagset_version=record.tagset_version or predefined_tagsets().get(record.tagset_namespace).version,
+                    tag=record.tag,
+                    features=record.features,
+                    props=record.props,
+                )
+                for record in tag_cache.with_target(target.identifier).iter_records()
+            ]
+
+            targets_count += 1
+            records_count += len(tag_records)
+
+            db_create_changing_model(db, DataTagTargetTable, tag_target, created_at=created_at)
+            db_tag_records = db_create_changing_models(db, DataTagRecordTable, tag_records, created_at=created_at)
+
+            tag_clips = [
+                DataTagClip(
+                    source_sqn=db_tag_record.sqn,
+                    vehicle_name=db_tag_record.vehicle_name,
+                    clip_dt=clip_dt,
+                    clip_duration_us=standard_clip_duration_us(),
+                    begin_offset_us=td_to_us(begin_dt - clip_dt),
+                    end_offset_us=td_to_us(end_dt - clip_dt),
+                    tagset_namespace=db_tag_record.tagset_namespace,
+                    tagset_version=db_tag_record.tagset_version,
+                    tag=db_tag_record.tag,
+                )
+                for db_tag_record in db_tag_records
+                for clip_dt, _, begin_dt, end_dt in populate_clip_ranges(db_tag_record.begin_dt, db_tag_record.end_dt)
+            ]
+
+            db_create_sequence_models(db, DataTagClipTable, tag_clips)
+
+        return TagCacheSummary(targets_count=targets_count, records_count=records_count)

plexus/pulse/demeter/app/api/user_endpoints.py

@@ -0,0 +1,78 @@
+import datetime
+
+import sqlalchemy.orm as sa_orm
+from fastapi import APIRouter, Depends
+from fastapi.params import Query
+from plexus.common.utils.dtutils import dt_utc_now
+from plexus.pulse.common.utils.apiutils import managed_db_session
+from plexus.pulse.common.utils.ormutils import (
+    db_create_revision_model,
+    db_create_revision_models,
+    db_expire_revision_model,
+    db_read_active_revision_model_of_record,
+    db_read_active_revision_models,
+    db_update_revision_model,
+)
+
+from plexus.pulse.demeter.app.core.database import make_session
+from plexus.pulse.demeter.app.models.registry.user import User
+from plexus.pulse.demeter.app.models.registry.user import UserTable
+
+router = APIRouter(prefix="/pulse_demeter/user", tags=["user"])
+
+
+@router.post("/user/", response_model=UserTable)
+def create_user(
+    user: User,
+    created_at: datetime.datetime = Query(default_factory=dt_utc_now),
+    db: sa_orm.Session = Depends(make_session),
+) -> UserTable:
+    with managed_db_session(db):
+        return db_create_revision_model(db, UserTable, user, created_at)
+
+
+@router.post("/users/", response_model=list[UserTable])
+def create_users(
+    users: list[User],
+    created_at: datetime.datetime = Query(default_factory=dt_utc_now),
+    db: sa_orm.Session = Depends(make_session),
+) -> list[UserTable]:
+    with managed_db_session(db):
+        return db_create_revision_models(db, UserTable, users, created_at)
+
+
+@router.get("/user/{record_sqn}", response_model=UserTable)
+def read_user(record_sqn: int, db: sa_orm.Session = Depends(make_session)) -> UserTable:
+    with managed_db_session(db):
+        return db_read_active_revision_model_of_record(db, UserTable, record_sqn)
+
+
+@router.get("/users/", response_model=list[UserTable])
+def read_users(
+    skip: int = Query(0),
+    limit: int = Query(1000),
+    db: sa_orm.Session = Depends(make_session),
+) -> list[UserTable]:
+    with managed_db_session(db):
+        return db_read_active_revision_models(db, UserTable, skip, limit)
+
+
+@router.put("/user/{record_sqn}", response_model=UserTable)
+def update_user(
+    record_sqn: int,
+    user: User,
+    updated_at: datetime.datetime = Query(default_factory=dt_utc_now),
+    db: sa_orm.Session = Depends(make_session),
+) -> UserTable:
+    with managed_db_session(db):
+        return db_update_revision_model(db, UserTable, user, record_sqn, updated_at)
+
+
+@router.delete("/user/{record_sqn}", response_model=UserTable)
+def delete_user(
+    record_sqn: int,
+    updated_at: datetime.datetime = Query(default_factory=dt_utc_now),
+    db: sa_orm.Session = Depends(make_session),
+) -> UserTable:
+    with managed_db_session(db):
+        return db_expire_revision_model(db, UserTable, record_sqn, updated_at)

plexus/pulse/demeter/app/core/database.py

@@ -0,0 +1,43 @@
+from collections.abc import Generator
+
+import sqlalchemy as sa
+import sqlalchemy.orm as sa_orm
+from pydantic_settings import BaseSettings
+
+from plexus.pulse.demeter.utils.config import config
+
+
+class Settings(BaseSettings):
+    host: str = config().get("pulse.apps.database", "host", "localhost")
+    port: int = config().getint("pulse.apps.database", "port", 5432)
+    username: str = config().get("pulse.apps.database", "username", "pulse")
+    password: str = config().get("pulse.apps.database", "password", "pulse")
+    database: str = config().get("pulse.apps.database", "database", "pulse-apps")
+
+    class Config:
+        env_prefix = "PULSE_APPS_DATABASE_"
+        env_file = ".env"
+
+
+settings = Settings()
+
+url = sa.URL.create(
+    drivername="postgresql+psycopg",
+    host=settings.host,
+    port=settings.port,
+    username=settings.username,
+    password=settings.password,
+    database=settings.database,
+)
+
+engine = sa.create_engine(url, echo=True, future=True)
+session_maker = sa_orm.sessionmaker(autocommit=False, autoflush=False, bind=engine, future=True, expire_on_commit=False)
+
+
+def make_session() -> Generator[sa_orm.Session, None, None]:
+    db = session_maker()
+    db.execute(sa.sql.text("SET TIMEZONE TO 'UTC'"))
+    try:
+        yield db
+    finally:
+        db.close()

plexus/pulse/demeter/app/core/vault.py

@@ -0,0 +1,29 @@
+from plexus.pulse.common.utils.s3utils import s3_make_client, s3_make_progressed_client
+from pydantic_settings import BaseSettings
+
+from plexus.pulse.demeter.utils.config import config
+
+
+class Settings(BaseSettings):
+    access_key_id: str = config().get("pulse.apps.vault", "access_key_id", "")
+    secret_access_key: str = config().get("pulse.apps.vault", "secret_access_key", "")
+    endpoint_url: str = config().get("pulse.apps.vault", "endpoint_url", "http://localhost:9000")
+
+    class Config:
+        env_prefix = "PULSE_APPS_VAULT_"
+        env_file = ".env"
+
+
+settings = Settings()
+
+
+def make_client():
+    return s3_make_client(access_key_id=settings.access_key_id,
+                          secret_access_key=settings.secret_access_key,
+                          endpoint_url=settings.endpoint_url)
+
+
+def make_progressed_client():
+    return s3_make_progressed_client(access_key_id=settings.access_key_id,
+                                     secret_access_key=settings.secret_access_key,
+                                     endpoint_url=settings.endpoint_url)

plexus/pulse/demeter/app/main.py

@@ -0,0 +1,16 @@
+from fastapi import Response
+from fastapi import status
+
+from plexus.pulse.demeter.app.api import make_api
+
+app = make_api()
+
+
+@app.get("/health")
+async def check_health():
+    return Response(status_code=status.HTTP_200_OK)
+
+
+@app.get("/live")
+async def check_liveness():
+    return Response(status_code=status.HTTP_200_OK)

plexus/pulse/demeter/app/models/__init__.py

@@ -0,0 +1,33 @@
+from typing import Any, Literal
+
+import pydantic as pdt
+
+__all__ = [
+    "Pagination",
+    "TableColumn",
+    "ColumnOrdering",
+    "ColumnFormatter",
+]
+
+
+class Pagination(pdt.BaseModel):
+    skip: int
+    limit: int
+    total: int
+
+
+class ColumnOrdering(pdt.BaseModel):
+    semantic: Literal["lexical", "numeric", "datetime", "semver"] = "lexical"
+    direction: Literal["ascending", "descending"] = "ascending"
+
+
+class ColumnFormatter(pdt.BaseModel):
+    syntax: Literal["python_print", "python_format", "python_fstring", "python_strftime"] = "python_print"
+    template: str
+    args: list[Any] | None = None
+
+
+class TableColumn(pdt.BaseModel):
+    name: str
+    ordering: ColumnOrdering | None = None
+    formatter: ColumnFormatter | None = None