kleinkram 0.50.2__tar.gz → 0.51.0__tar.gz

{kleinkram-0.50.2 → kleinkram-0.51.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kleinkram
-Version: 0.50.2
+Version: 0.51.0
 Summary: give me your bags
 Author: Cyrill Püntener, Dominique Garmier, Johann Schwabe
 Author-email: pucyril@ethz.ch, dgarmier@ethz.ch, jschwab@ethz.ch

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/api/deser.py

@@ -4,6 +4,7 @@ from datetime import datetime
 from enum import Enum
 from typing import Any
 from typing import Dict
+from typing import List
 from typing import Literal
 from typing import NewType
 from typing import Tuple
@@ -14,6 +15,7 @@ import dateutil.parser
 from kleinkram.errors import ParsingError
 from kleinkram.models import File
 from kleinkram.models import FileState
+from kleinkram.models import MetadataValue
 from kleinkram.models import Mission
 from kleinkram.models import Project
 
@@ -51,6 +53,7 @@ class MissionObjectKeys(str, Enum):
     DESCRIPTION = "description"
     CREATED_AT = "createdAt"
     UPDATED_AT = "updatedAt"
+    TAGS = "tags"
 
 
 class ProjectObjectKeys(str, Enum):
@@ -59,6 +62,7 @@ class ProjectObjectKeys(str, Enum):
     DESCRIPTION = "description"
     CREATED_AT = "createdAt"
     UPDATED_AT = "updatedAt"
+    REQUIRED_TAGS = "requiredTags"
 
 
 def _get_nested_info(data, key: Literal["mission", "project"]) -> Tuple[UUID, str]:
@@ -83,6 +87,25 @@ def _parse_file_state(state: str) -> FileState:
         raise ParsingError(f"error parsing file state: {state}") from e
 
 
+def _parse_metadata(tags: List[Dict]) -> Dict[str, MetadataValue]:
+    result = {}
+    try:
+        for tag in tags:
+            entry = {
+                tag.get("name"): MetadataValue(
+                    tag.get("valueAsString"), tag.get("datatype")
+                )
+            }
+            result.update(entry)
+        return result
+    except ValueError as e:
+        raise ParsingError(f"error parsing metadata: {e}") from e
+
+
+def _parse_required_tags(tags: List[Dict]) -> list[str]:
+    return list(_parse_metadata(tags).keys())
+
+
 def _parse_project(project_object: ProjectObject) -> Project:
     try:
         id_ = UUID(project_object[ProjectObjectKeys.UUID], version=4)
@@ -90,6 +113,9 @@ def _parse_project(project_object: ProjectObject) -> Project:
         description = project_object[ProjectObjectKeys.DESCRIPTION]
         created_at = _parse_datetime(project_object[ProjectObjectKeys.CREATED_AT])
         updated_at = _parse_datetime(project_object[ProjectObjectKeys.UPDATED_AT])
+        required_tags = _parse_required_tags(
+            project_object[ProjectObjectKeys.REQUIRED_TAGS]
+        )
     except Exception as e:
         raise ParsingError(f"error parsing project: {project_object}") from e
     return Project(
@@ -98,6 +124,7 @@ def _parse_project(project_object: ProjectObject) -> Project:
         description=description,
         created_at=created_at,
         updated_at=updated_at,
+        required_tags=required_tags,
     )
 
 
@@ -107,7 +134,7 @@ def _parse_mission(mission: MissionObject) -> Mission:
         name = mission[MissionObjectKeys.NAME]
         created_at = _parse_datetime(mission[MissionObjectKeys.CREATED_AT])
         updated_at = _parse_datetime(mission[MissionObjectKeys.UPDATED_AT])
-        metadata = {}  # TODO: this crap is really bad to parse
+        metadata = _parse_metadata(mission[MissionObjectKeys.TAGS])
 
         project_id, project_name = _get_nested_info(mission, PROJECT)
 

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/api/file_transfer.py

@@ -572,19 +572,20 @@ def upload_files(
 
         avg_speed_bps = total_uploaded_bytes / elapsed_time if elapsed_time > 0 else 0
 
-        console.print(f"Upload took {elapsed_time:.2f} seconds")
-        console.print(f"Total uploaded: {format_bytes(total_uploaded_bytes)}")
-        console.print(f"Average speed: {format_bytes(avg_speed_bps, speed=True)}")
-
-        if failed_files > 0:
-            console.print(
-                f"\nUploaded {len(files) - failed_files - skipped_files} files, {skipped_files} skipped, {failed_files} uploads failed",
-                style="red",
-            )
-        else:
-            console.print(
-                f"\nUploaded {len(files) - skipped_files} files, {skipped_files} skipped"
-            )
+        if verbose:
+            console.print(f"Upload took {elapsed_time:.2f} seconds")
+            console.print(f"Total uploaded: {format_bytes(total_uploaded_bytes)}")
+            console.print(f"Average speed: {format_bytes(avg_speed_bps, speed=True)}")
+
+            if failed_files > 0:
+                console.print(
+                    f"\nUploaded {len(files) - failed_files - skipped_files} files, {skipped_files} skipped, {failed_files} uploads failed",
+                    style="red",
+                )
+            else:
+                console.print(
+                    f"\nUploaded {len(files) - skipped_files} files, {skipped_files} skipped"
+                )
 
 
 def download_files(

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/api/pagination.py

@@ -17,6 +17,7 @@ DataPage = Dict[str, Any]
 PAGE_SIZE = 128
 SKIP = "skip"
 TAKE = "take"
+EXACT_MATCH = "exactMatch"
 
 
 def paginated_request(
@@ -25,6 +26,7 @@ def paginated_request(
     params: Optional[Mapping[str, Any]] = None,
     max_entries: Optional[int] = None,
     page_size: int = PAGE_SIZE,
+    exact_match: bool = False,
 ) -> Generator[DataPage, None, None]:
     total_entries_count = 0
 
@@ -32,6 +34,7 @@ def paginated_request(
 
     params[TAKE] = page_size
     params[SKIP] = 0
+    params[EXACT_MATCH] = str(exact_match).lower()  # pass string rather than bool
 
     while True:
         resp = client.get(endpoint, params=params)

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/api/routes.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
 import json
+import tempfile
 from enum import Enum
+from pathlib import Path
 from typing import Any
 from typing import Dict
 from typing import Generator
@@ -38,12 +40,14 @@ from kleinkram.errors import InvalidMissionQuery
 from kleinkram.errors import InvalidProjectQuery
 from kleinkram.errors import MissionExists
 from kleinkram.errors import MissionNotFound
+from kleinkram.errors import MissionValidationError
 from kleinkram.errors import ProjectExists
 from kleinkram.errors import ProjectNotFound
 from kleinkram.models import File
 from kleinkram.models import Mission
 from kleinkram.models import Project
 from kleinkram.utils import is_valid_uuid4
+from kleinkram.utils import split_args
 
 __all__ = [
     "_get_api_version",
@@ -159,15 +163,22 @@ def get_projects(
     client: AuthenticatedClient,
     project_query: ProjectQuery,
     max_entries: Optional[int] = None,
+    exact_match: bool = False,
 ) -> Generator[Project, None, None]:
     params = _project_query_to_params(project_query)
     response_stream = paginated_request(
-        client, PROJECT_ENDPOINT, params=params, max_entries=max_entries
+        client,
+        PROJECT_ENDPOINT,
+        params=params,
+        max_entries=max_entries,
+        exact_match=exact_match,
     )
     yield from map(lambda p: _parse_project(ProjectObject(p)), response_stream)
 
 
-def get_project(client: AuthenticatedClient, query: ProjectQuery) -> Project:
+def get_project(
+    client: AuthenticatedClient, query: ProjectQuery, exact_match: bool = False
+) -> Project:
     """\
     get a unique project by specifying a project spec
     """
@@ -176,7 +187,7 @@ def get_project(client: AuthenticatedClient, query: ProjectQuery) -> Project:
             f"Project query does not uniquely determine project: {query}"
         )
     try:
-        return next(get_projects(client, query))
+        return next(get_projects(client, query, exact_match=exact_match))
     except StopIteration:
         raise ProjectNotFound(f"Project not found: {query}")
 
@@ -220,6 +231,28 @@ def _mission_name_is_available(
     return False
 
 
+def _validate_mission_name(
+    client: AuthenticatedClient, project_id: UUID, mission_name: str
+) -> None:
+    if not _mission_name_is_available(client, mission_name, project_id):
+        raise MissionExists(
+            f"Mission with name: `{mission_name}` already exists"
+            f" in project: {project_id}"
+        )
+
+    if is_valid_uuid4(mission_name):
+        raise ValueError(
+            f"Mission name: `{mission_name}` is a valid UUIDv4, "
+            "mission names must not be valid UUIDv4's"
+        )
+
+    if mission_name.endswith(" "):
+        raise ValueError(
+            "A mission name cannot end with a whitespace. "
+            f"The given mission name was '{mission_name}'"
+        )
+
+
 def _project_name_is_available(client: AuthenticatedClient, project_name: str) -> bool:
     project_query = ProjectQuery(patterns=[project_name])
     try:
@@ -229,6 +262,50 @@ def _project_name_is_available(client: AuthenticatedClient, project_name: str) -
     return False
 
 
+def _validate_mission_created(
+    client: AuthenticatedClient, project_id: str, mission_name: str
+) -> None:
+    """
+    validate that a mission is successfully created
+    """
+    mission_ids, mission_patterns = split_args([mission_name])
+    project_ids, project_patterns = split_args([project_id])
+
+    project_query = ProjectQuery(ids=project_ids, patterns=project_patterns)
+    mission_query = MissionQuery(
+        ids=mission_ids,
+        patterns=mission_patterns,
+        project_query=project_query,
+    )
+    try:
+        with tempfile.NamedTemporaryFile(suffix=".mcap", delete=False) as tmp:
+            tmp.write(b"dummy content")
+            tmp_path = Path(tmp.name)
+
+        kleinkram.core.upload(
+            client=client,
+            query=mission_query,
+            file_paths=[tmp_path],
+            verbose=False,
+        )
+
+        file_query = FileQuery(
+            ids=[],
+            patterns=[tmp_path.name],
+            mission_query=mission_query,
+        )
+        file_parsed = get_file(client, file_query)
+
+        kleinkram.core.delete_files(client=client, file_ids=[file_parsed.id])
+
+    except Exception as e:
+        raise MissionValidationError(f"Mission validation failed: {e}")
+
+    finally:
+        if tmp_path.exists():
+            tmp_path.unlink()
+
+
 def _create_mission(
     client: AuthenticatedClient,
     project_id: UUID,
@@ -236,6 +313,7 @@ def _create_mission(
     *,
     metadata: Optional[Dict[str, str]] = None,
     ignore_missing_tags: bool = False,
+    required_tags: Optional[List[str]] = None,
 ) -> UUID:
     """\
     creates a new mission with the given name and project_id
@@ -246,16 +324,11 @@ def _create_mission(
     if metadata is None:
         metadata = {}
 
-    if not _mission_name_is_available(client, mission_name, project_id):
-        raise MissionExists(
-            f"Mission with name: `{mission_name}` already exists"
-            f" in project: {project_id}"
-        )
+    _validate_mission_name(client, project_id, mission_name)
 
-    if is_valid_uuid4(mission_name):
-        raise ValueError(
-            f"Mission name: `{mission_name}` is a valid UUIDv4, "
-            "mission names must not be valid UUIDv4's"
+    if required_tags and not set(required_tags).issubset(metadata.keys()):
+        raise InvalidMissionMetadata(
+            f"Mission tags `{required_tags}` are required but missing from metadata: {metadata}"
         )
 
     # we need to translate tag keys to tag type ids
@@ -267,9 +340,9 @@ def _create_mission(
         "tags": {str(k): v for k, v in tags.items()},
         "ignoreTags": ignore_missing_tags,
     }
-
     resp = client.post(CREATE_MISSION, json=payload)
     resp.raise_for_status()
+    _validate_mission_created(client, str(project_id), mission_name)
 
     return UUID(resp.json()["uuid"], version=4)
 
@@ -288,18 +361,37 @@ def _create_project(
     return UUID(resp.json()["uuid"], version=4)
 
 
+def _validate_tag_value(tag_value, tag_datatype) -> None:
+    if tag_datatype == "NUMBER":
+        try:
+            float(tag_value)
+        except ValueError:
+            raise InvalidMissionMetadata(f"Value '{tag_value}' is not a valid NUMBER")
+    elif tag_datatype == "BOOLEAN":
+        if tag_value.lower() not in {"true", "false"}:
+            raise InvalidMissionMetadata(
+                f"Value '{tag_value}' is not a valid BOOLEAN (expected 'true' or 'false')"
+            )
+    else:
+        pass  # any string is fine
+    # TODO: add check for LOCATION tag datatype
+
+
 def _get_metadata_type_id_by_name(
     client: AuthenticatedClient, tag_name: str
-) -> Optional[UUID]:
+) -> Tuple[Optional[UUID], str]:
     resp = client.get(TAG_TYPE_BY_NAME, params={"name": tag_name, "take": 1})
 
     if resp.status_code in (403, 404):
         return None
 
     resp.raise_for_status()
+    try:
+        data = resp.json()["data"][0]
+    except IndexError:
+        return None, None
 
-    data = resp.json()[0]
-    return UUID(data["uuid"], version=4)
+    return UUID(data["uuid"], version=4), data["datatype"]
 
 
 def _get_tags_map(
@@ -309,9 +401,10 @@ def _get_tags_map(
     # why are we using metadata type ids as keys???
     ret = {}
     for key, val in metadata.items():
-        metadata_type_id = _get_metadata_type_id_by_name(client, key)
+        metadata_type_id, tag_datatype = _get_metadata_type_id_by_name(client, key)
         if metadata_type_id is None:
             raise InvalidMissionMetadata(f"metadata field: {key} does not exist")
+        _validate_tag_value(val, tag_datatype)
         ret[metadata_type_id] = val
     return ret
 

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/cli/_mission.py

@@ -13,6 +13,7 @@ from kleinkram.api.query import ProjectQuery
 from kleinkram.api.routes import get_mission
 from kleinkram.api.routes import get_project
 from kleinkram.config import get_shared_state
+from kleinkram.errors import InvalidMissionQuery
 from kleinkram.printing import print_mission_info
 from kleinkram.utils import load_metadata
 from kleinkram.utils import split_args
@@ -45,13 +46,16 @@ def create(
     metadata_dct = load_metadata(Path(metadata)) if metadata else {}  # noqa
 
     client = AuthenticatedClient()
-    project_id = get_project(client, project_query).id
+    project = get_project(client, project_query)
+    project_id = project.id
+    project_required_tags = project.required_tags
     mission_id = kleinkram.api.routes._create_mission(
         client,
         project_id,
         mission_name,
         metadata=metadata_dct,
         ignore_missing_tags=ignore_missing_tags,
+        required_tags=project_required_tags,
     )
 
     mission_parsed = get_mission(client, MissionQuery(ids=[mission_id]))
@@ -120,9 +124,6 @@ def delete(
         False, "--confirm", "-y", "--yes", help="confirm deletion"
     ),
 ) -> None:
-    if not confirm:
-        typer.confirm(f"delete {project} {mission}", abort=True)
-
     project_ids, project_patterns = split_args([project] if project else [])
     project_query = ProjectQuery(ids=project_ids, patterns=project_patterns)
 
@@ -132,9 +133,20 @@ def delete(
         patterns=mission_patterns,
         project_query=project_query,
     )
+    if mission_patterns and not (project_patterns or project_ids):
+        raise InvalidMissionQuery(
+            "Mission query does not uniquely determine mission. "
+            "Project name or id must be specified when deleting by mission name"
+        )
 
     client = AuthenticatedClient()
     mission_parsed = get_mission(client, mission_query)
+    if not confirm:
+        if project:
+            typer.confirm(f"delete {project} {mission}", abort=True)
+        else:
+            typer.confirm(f"delete {mission_parsed.name} {mission}", abort=True)
+
     kleinkram.core.delete_mission(client=client, mission_id=mission_parsed.id)
 
 

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/cli/_project.py

@@ -90,7 +90,7 @@ def delete(
     project_query = ProjectQuery(ids=project_ids, patterns=project_patterns)
 
     client = AuthenticatedClient()
-    project_id = get_project(client=client, query=project_query).id
+    project_id = get_project(client=client, query=project_query, exact_match=True).id
     kleinkram.core.delete_project(client=client, project_id=project_id)
 
 

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/cli/_upload.py

@@ -13,6 +13,7 @@ from kleinkram.api.query import MissionQuery
 from kleinkram.api.query import ProjectQuery
 from kleinkram.config import get_shared_state
 from kleinkram.errors import FileNameNotSupported
+from kleinkram.errors import DatatypeNotSupported
 from kleinkram.errors import MissionNotFound
 from kleinkram.utils import load_metadata
 from kleinkram.utils import split_args
@@ -44,6 +45,9 @@ def upload(
         False,
         help="fix filenames before upload, this does not change the filenames locally",
     ),
+    experimental_datatypes: bool = typer.Option(
+        False, help="allow experimental datatypes (yaml, svo2, db3, tum)"
+    ),
     ignore_missing_tags: bool = typer.Option(False, help="ignore mission tags"),
 ) -> None:
     # get filepaths
@@ -61,6 +65,17 @@ def upload(
 
     if not fix_filenames:
         for file in file_paths:
+
+            # check for experimental datatypes and throw an exception if not allowed
+            EXPERIMENTAL_DATATYPES = {".yaml", ".svo2", ".db3", ".tum"}
+
+            if not experimental_datatypes:
+                if file.suffix.lower() in EXPERIMENTAL_DATATYPES:
+                    raise DatatypeNotSupported(
+                        f"Datatype '{file.suffix}' for file {file} is not supported without the "
+                        f"`--experimental-datatypes` flag. "
+                    )
+
             if not kleinkram.utils.check_filename_is_sanatized(file.stem):
                 raise FileNameNotSupported(
                     f"Only `{''.join(kleinkram.utils.INTERNAL_ALLOWED_CHARS)}` are "

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/core.py

@@ -107,9 +107,9 @@ def upload(
 
     if create and mission is None:
         # check if project exists and get its id at the same time
-        project_id = kleinkram.api.routes.get_project(
-            client, query=query.project_query
-        ).id
+        project = kleinkram.api.routes.get_project(client, query=query.project_query)
+        project_id = project.id
+        project_required_tags = project.required_tags
         mission_name = check_mission_query_is_creatable(query)
         kleinkram.api.routes._create_mission(
             client,
@@ -117,6 +117,7 @@ def upload(
             mission_name,
             metadata=metadata or {},
             ignore_missing_tags=ignore_missing_metadata,
+            required_tags=project_required_tags,
         )
         mission = kleinkram.api.routes.get_mission(client, query)
 
@@ -281,7 +282,9 @@ def delete_mission(*, client: AuthenticatedClient, mission_id: UUID) -> None:
 
 def delete_project(*, client: AuthenticatedClient, project_id: UUID) -> None:
     pquery = ProjectQuery(ids=[project_id])
-    _ = kleinkram.api.routes.get_project(client, pquery)  # check if project exists
+    _ = kleinkram.api.routes.get_project(
+        client, pquery, exact_match=True
+    )  # check if project exists
 
     # delete all missions and files
     missions = list(

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/errors.py

@@ -43,9 +43,15 @@ class FileTypeNotSupported(Exception): ...
 class FileNameNotSupported(Exception): ...
 
 
+class DatatypeNotSupported(Exception): ...
+
+
 class InvalidMissionMetadata(Exception): ...
 
 
+class MissionValidationError(Exception): ...
+
+
 class NotAuthenticated(Exception):
     def __init__(self) -> None:
         super().__init__(LOGIN_MESSAGE)

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/models.py

@@ -41,6 +41,7 @@ class Project:
     description: str
     created_at: datetime
     updated_at: datetime
+    required_tags: List[str]
 
 
 @dataclass(frozen=True)

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/printing.py

@@ -233,7 +233,7 @@ def file_info_table(file: File) -> Table:
 
 
 def mission_info_table(
-    mission: Mission, print_metadata: bool = False
+    mission: Mission, print_metadata: bool = True
 ) -> Tuple[Table, ...]:
     table = Table("k", "v", title=f"mission info: {mission.name}", show_header=False)
 
@@ -269,6 +269,7 @@ def project_info_table(project: Project) -> Table:
     table.add_row("description", project.description)
     table.add_row("created", str(project.created_at))
     table.add_row("updated", str(project.updated_at))
+    table.add_row("required tags", ", ".join(project.required_tags))
 
     return table
 

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram/utils.py

@@ -28,10 +28,7 @@ from kleinkram.types import IdLike
 from kleinkram.types import PathLike
 
 INTERNAL_ALLOWED_CHARS = string.ascii_letters + string.digits + "_" + "-"
-SUPPORT_FILE_TYPES = [
-    ".bag",
-    ".mcap",
-]
+SUPPORT_FILE_TYPES = [".bag", ".mcap", ".db3", ".svo2", ".tum", ".yaml"]
 
 
 def file_paths_from_files(

{kleinkram-0.50.2 → kleinkram-0.51.0}/kleinkram.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kleinkram
-Version: 0.50.2
+Version: 0.51.0
 Summary: give me your bags
 Author: Cyrill Püntener, Dominique Garmier, Johann Schwabe
 Author-email: pucyril@ethz.ch, dgarmier@ethz.ch, jschwab@ethz.ch

{kleinkram-0.50.2 → kleinkram-0.51.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = kleinkram
-version = 0.50.2
+version = 0.51.0
 description = give me your bags
 long_description = file: README.md
 long_description_content_type = text/markdown