kleinkram 0.38.1.dev20241212075157__py3-none-any.whl → 0.38.1.dev20250207122632__py3-none-any.whl

kleinkram/api/routes.py

@@ -1,121 +1,230 @@
 from __future__ import annotations
 
+import json
+from enum import Enum
+from typing import Any
 from typing import Dict
+from typing import Generator
 from typing import List
 from typing import Optional
+from typing import Sequence
 from typing import Tuple
 from uuid import UUID
 
 import httpx
+
+import kleinkram.errors
 from kleinkram.api.client import AuthenticatedClient
-from kleinkram.api.parsing import _parse_file
-from kleinkram.api.parsing import _parse_mission
-from kleinkram.api.parsing import _parse_project
-from kleinkram.config import Config
+from kleinkram.api.deser import FileObject
+from kleinkram.api.deser import MissionObject
+from kleinkram.api.deser import ProjectObject
+from kleinkram.api.deser import _parse_file
+from kleinkram.api.deser import _parse_mission
+from kleinkram.api.deser import _parse_project
+from kleinkram.api.pagination import paginated_request
+from kleinkram.api.query import FileQuery
+from kleinkram.api.query import MissionQuery
+from kleinkram.api.query import ProjectQuery
+from kleinkram.api.query import file_query_is_unique
+from kleinkram.api.query import mission_query_is_unique
+from kleinkram.api.query import project_query_is_unique
+from kleinkram.config import get_config
 from kleinkram.errors import AccessDenied
+from kleinkram.errors import InvalidFileQuery
+from kleinkram.errors import InvalidMissionMetadata
+from kleinkram.errors import InvalidMissionQuery
+from kleinkram.errors import InvalidProjectQuery
 from kleinkram.errors import MissionExists
 from kleinkram.errors import MissionNotFound
-from kleinkram.models import DataType
+from kleinkram.errors import ProjectExists
+from kleinkram.errors import ProjectNotFound
 from kleinkram.models import File
 from kleinkram.models import Mission
 from kleinkram.models import Project
-from kleinkram.models import TagType
 from kleinkram.utils import is_valid_uuid4
 
 __all__ = [
-    "_get_projects",
-    "_get_missions_by_project",
-    "_get_files_by_mission",
-    "_create_mission",
-    "_update_mission_metadata",
     "_get_api_version",
     "_claim_admin",
+    "_create_mission",
+    "_create_project",
+    "_update_mission",
+    "_update_project",
+    "_delete_files",
+    "_delete_mission",
+    "_delete_project",
+    "get_files",
+    "get_missions",
+    "get_projects",
+    "get_project",
+    "get_mission",
+    "get_file",
 ]
 
 
-MAX_PAGINATION = 10_000
-
 CLAIM_ADMIN = "/user/claimAdmin"
-PROJECT_ALL = "/project/filtered"
-MISSIONS_BY_PROJECT = "/mission/filtered"
-MISSION_BY_NAME = "/mission/byName"
-MISSION_CREATE = "/mission/create"
-MISSION_UPDATE_METADATA = "/mission/tags"
-FILE_OF_MISSION = "/file/ofMission"
-TAG_TYPE_BY_NAME = "/tag/filtered"
 GET_STATUS = "/user/me"
 
+UPDATE_PROJECT = "/project"
+UPDATE_MISSION = "/mission/tags"  # TODO: just metadata for now
+CREATE_MISSION = "/mission/create"
+CREATE_PROJECT = "/project/create"
 
-def _get_projects(client: AuthenticatedClient) -> list[Project]:
-    resp = client.get(PROJECT_ALL)
-
-    if resp.status_code in (403, 404):
-        return []
-
-    resp.raise_for_status()
-
-    ret = []
-    for pr in resp.json()[0]:
-        ret.append(_parse_project(pr))
-
-    return ret
-
-
-def _get_missions_by_project(
-    client: AuthenticatedClient, project: Project
-) -> List[Mission]:
-    params = {"uuid": str(project.id), "take": MAX_PAGINATION}
 
-    resp = client.get(MISSIONS_BY_PROJECT, params=params)
+FILE_ENDPOINT = "/file/many"
+MISSION_ENDPOINT = "/mission/many"
+PROJECT_ENDPOINT = "/project/many"
 
-    if resp.status_code in (403, 404):
-        return []
-
-    resp.raise_for_status()
-
-    data = resp.json()
-    missions = []
+TAG_TYPE_BY_NAME = "/tag/filtered"
 
-    for mission in data[0]:
-        missions.append(_parse_mission(mission, project))
 
-    return missions
+class Params(str, Enum):
+    FILE_PATTERNS = "filePatterns"
+    FILE_IDS = "fileUuids"
+    MISSION_PATTERNS = "missionPatterns"
+    MISSION_IDS = "missionUuids"
+    PROJECT_PATTERNS = "projectPatterns"
+    PROJECT_IDS = "projectUuids"
 
 
-def _get_files_by_mission(client: AuthenticatedClient, mission: Mission) -> List[File]:
-    params = {"uuid": str(mission.id), "take": MAX_PAGINATION}
+def _handle_list_params(params: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    json dumps lists
+    """
+    new_params = {}
+    for k, v in params.items():
+        if not isinstance(v, list):
+            new_params[k] = v
+        else:
+            new_params[k] = json.dumps(v)
+    return new_params
+
+
+def _project_query_to_params(
+    project_query: ProjectQuery,
+) -> Dict[str, List[str]]:
+    params = {}
+    if project_query.patterns:
+        params[Params.PROJECT_PATTERNS.value] = project_query.patterns
+    if project_query.ids:
+        params[Params.PROJECT_IDS.value] = list(map(str, project_query.ids))
+    return params
+
+
+def _mission_query_to_params(mission_query: MissionQuery) -> Dict[str, List[str]]:
+    params = _project_query_to_params(mission_query.project_query)
+    if mission_query.patterns:
+        params[Params.MISSION_PATTERNS.value] = mission_query.patterns
+    if mission_query.ids:
+        params[Params.MISSION_IDS.value] = list(map(str, mission_query.ids))
+    return params
+
+
+def _file_query_to_params(file_query: FileQuery) -> Dict[str, List[str]]:
+    params = _mission_query_to_params(file_query.mission_query)
+    if file_query.patterns:
+        params[Params.FILE_PATTERNS.value] = list(file_query.patterns)
+    if file_query.ids:
+        params[Params.FILE_IDS.value] = list(map(str, file_query.ids))
+    return params
+
+
+def get_files(
+    client: AuthenticatedClient,
+    file_query: FileQuery,
+    max_entries: Optional[int] = None,
+) -> Generator[File, None, None]:
+    params = _file_query_to_params(file_query)
+    response_stream = paginated_request(
+        client, FILE_ENDPOINT, params=params, max_entries=max_entries
+    )
+    yield from map(lambda f: _parse_file(FileObject(f)), response_stream)
 
-    resp = client.get(FILE_OF_MISSION, params=params)
 
-    if resp.status_code in (403, 404):
-        return []
+def get_missions(
+    client: AuthenticatedClient,
+    mission_query: MissionQuery,
+    max_entries: Optional[int] = None,
+) -> Generator[Mission, None, None]:
+    params = _mission_query_to_params(mission_query)
+    response_stream = paginated_request(
+        client, MISSION_ENDPOINT, params=params, max_entries=max_entries
+    )
+    yield from map(lambda m: _parse_mission(MissionObject(m)), response_stream)
 
-    resp.raise_for_status()
 
-    data = resp.json()
+def get_projects(
+    client: AuthenticatedClient,
+    project_query: ProjectQuery,
+    max_entries: Optional[int] = None,
+) -> Generator[Project, None, None]:
+    params = _project_query_to_params(project_query)
+    response_stream = paginated_request(
+        client, PROJECT_ENDPOINT, params=params, max_entries=max_entries
+    )
+    yield from map(lambda p: _parse_project(ProjectObject(p)), response_stream)
 
-    files = []
-    for file in data[0]:
-        files.append(_parse_file(file, mission))
 
-    return files
+def get_project(client: AuthenticatedClient, query: ProjectQuery) -> Project:
+    """\
+    get a unique project by specifying a project spec
+    """
+    if not project_query_is_unique(query):
+        raise InvalidProjectQuery(
+            f"Project query does not uniquely determine project: {query}"
+        )
+    try:
+        return next(get_projects(client, query))
+    except StopIteration:
+        raise ProjectNotFound(f"Project not found: {query}")
 
 
-def _get_mission_id_by_name(
-    client: AuthenticatedClient, mission_name, project_id: UUID
-) -> Optional[UUID]:
-    params = {"name": mission_name, "projectUUID": str(project_id)}
-    resp = client.get(MISSION_BY_NAME, params=params)
+def get_mission(client: AuthenticatedClient, query: MissionQuery) -> Mission:
+    """\
+    get a unique mission by specifying a mission query
+    """
+    if not mission_query_is_unique(query):
+        raise InvalidMissionQuery(
+            f"Mission query does not uniquely determine mission: {query}"
+        )
+    try:
+        return next(get_missions(client, query))
+    except StopIteration:
+        raise MissionNotFound(f"Mission not found: {query}")
 
-    if resp.status_code in (403, 404):
-        return None
 
-    # TODO: handle other status codes
-    resp.raise_for_status()
+def get_file(client: AuthenticatedClient, query: FileQuery) -> File:
+    """\
+    get a unique file by specifying a file query
+    """
+    if not file_query_is_unique(query):
+        raise InvalidFileQuery(f"File query does not uniquely determine file: {query}")
+    try:
+        return next(get_files(client, query))
+    except StopIteration:
+        raise kleinkram.errors.FileNotFound(f"File not found: {query}")
+
+
+def _mission_name_is_available(
+    client: AuthenticatedClient, mission_name: str, project_id: UUID
+) -> bool:
+    mission_query = MissionQuery(
+        patterns=[mission_name], project_query=ProjectQuery(ids=[project_id])
+    )
+    try:
+        _ = get_mission(client, mission_query)
+    except MissionNotFound:
+        return True
+    return False
 
-    data = resp.json()
 
-    return UUID(data["uuid"], version=4)
+def _project_name_is_available(client: AuthenticatedClient, project_name: str) -> bool:
+    project_query = ProjectQuery(patterns=[project_name])
+    try:
+        _ = get_project(client, project_query)
+    except ProjectNotFound:
+        return True
+    return False
 
 
 def _create_mission(
@@ -135,8 +244,11 @@ def _create_mission(
     if metadata is None:
         metadata = {}
 
-    if _get_mission_id_by_name(client, mission_name, project_id) is not None:
-        raise MissionExists(f"Mission with name: `{mission_name}` already exists")
+    if not _mission_name_is_available(client, mission_name, project_id):
+        raise MissionExists(
+            f"Mission with name: `{mission_name}` already exists"
+            f" in project: {project_id}"
+        )
 
     if is_valid_uuid4(mission_name):
         raise ValueError(
@@ -154,15 +266,29 @@ def _create_mission(
         "ignoreTags": ignore_missing_tags,
     }
 
-    resp = client.post(MISSION_CREATE, json=payload)
+    resp = client.post(CREATE_MISSION, json=payload)
     resp.raise_for_status()
 
     return UUID(resp.json()["uuid"], version=4)
 
 
-def _get_tag_type_by_name(
+def _create_project(
+    client: AuthenticatedClient, project_name: str, description: str
+) -> UUID:
+    if not _project_name_is_available(client, project_name):
+        raise ProjectExists(f"Project with name: `{project_name}` already exists")
+
+    # TODO: check name and description are valid
+    payload = {"name": project_name, "description": description}
+    resp = client.post(CREATE_PROJECT, json=payload)
+    resp.raise_for_status()
+
+    return UUID(resp.json()["uuid"], version=4)
+
+
+def _get_metadata_type_id_by_name(
     client: AuthenticatedClient, tag_name: str
-) -> Optional[TagType]:
+) -> Optional[UUID]:
     resp = client.get(TAG_TYPE_BY_NAME, params={"name": tag_name, "take": 1})
 
     if resp.status_code in (403, 404):
@@ -171,33 +297,24 @@ def _get_tag_type_by_name(
     resp.raise_for_status()
 
     data = resp.json()[0]
-    tag_type = TagType(
-        name=data["name"],
-        id=UUID(data["uuid"], version=4),
-        data_type=DataType(data["datatype"]),
-        description=data["description"],
-    )
-    return tag_type
+    return UUID(data["uuid"], version=4)
 
 
 def _get_tags_map(
     client: AuthenticatedClient, metadata: Dict[str, str]
 ) -> Dict[UUID, str]:
     # TODO: this needs a better endpoint
+    # why are we using metadata type ids as keys???
     ret = {}
     for key, val in metadata.items():
-        tag_type = _get_tag_type_by_name(client, key)
-
-        if tag_type is None:
-            print(f"tag: {key} not found")
-            continue
-
-        ret[tag_type.id] = val
-
+        metadata_type_id = _get_metadata_type_id_by_name(client, key)
+        if metadata_type_id is None:
+            raise InvalidMissionMetadata(f"metadata field: {key} does not exist")
+        ret[metadata_type_id] = val
     return ret
 
 
-def _update_mission_metadata(
+def _update_mission(
     client: AuthenticatedClient, mission_id: UUID, *, metadata: Dict[str, str]
 ) -> None:
     tags_dct = _get_tags_map(client, metadata)
@@ -205,22 +322,40 @@ def _update_mission_metadata(
         "missionUUID": str(mission_id),
         "tags": {str(k): v for k, v in tags_dct.items()},
     }
-    resp = client.post(MISSION_UPDATE_METADATA, json=payload)
+    resp = client.post(UPDATE_MISSION, json=payload)
 
     if resp.status_code == 404:
         raise MissionNotFound
-
     if resp.status_code == 403:
         raise AccessDenied(f"cannot update mission: {mission_id}")
 
     resp.raise_for_status()
 
 
+def _update_project(
+    client: AuthenticatedClient,
+    project_id: UUID,
+    *,
+    description: Optional[str] = None,
+    new_name: Optional[str] = None,
+) -> None:
+    if description is None and new_name is None:
+        raise ValueError("either description or new_name must be provided")
+
+    body = {}
+    if description is not None:
+        body["description"] = description
+    if new_name is not None:
+        body["name"] = new_name
+    resp = client.put(f"{UPDATE_PROJECT}/{project_id}", json=body)
+    resp.raise_for_status()
+
+
 def _get_api_version() -> Tuple[int, int, int]:
-    config = Config()
+    config = get_config()
     client = httpx.Client()
 
-    resp = client.get(f"{config.endpoint}{GET_STATUS}")
+    resp = client.get(f"{config.endpoint.api}{GET_STATUS}")
     vers = resp.headers["kleinkram-version"].split(".")
 
     return tuple(map(int, vers))  # type: ignore
@@ -233,3 +368,37 @@ def _claim_admin(client: AuthenticatedClient) -> None:
     response = client.post(CLAIM_ADMIN)
     response.raise_for_status()
     return
+
+
+FILE_DELETE_MANY = "/file/deleteMultiple"
+
+
+def _delete_files(
+    client: AuthenticatedClient, file_ids: Sequence[UUID], mission_id: UUID
+) -> None:
+    payload = {
+        "uuids": [str(file_id) for file_id in file_ids],
+        "missionUUID": str(mission_id),
+    }
+    resp = client.post(FILE_DELETE_MANY, json=payload)
+    resp.raise_for_status()
+
+
+MISSION_DELETE_ONE = "/mission/{}"
+
+
+def _delete_mission(client: AuthenticatedClient, mission_id: UUID) -> None:
+    resp = client.delete(MISSION_DELETE_ONE.format(mission_id))
+
+    # 409 is returned if the mission has files
+    # 403 is returned if the mission does not exist / user cant delete
+
+    resp.raise_for_status()
+
+
+PROJECT_DELETE_ONE = "/project/{}"
+
+
+def _delete_project(client: AuthenticatedClient, project_id: UUID) -> None:
+    resp = client.delete(PROJECT_DELETE_ONE.format(project_id))
+    resp.raise_for_status()

kleinkram/auth.py

@@ -7,9 +7,10 @@ from http.server import BaseHTTPRequestHandler
 from http.server import HTTPServer
 from typing import Optional
 
-from kleinkram.config import Config
 from kleinkram.config import CONFIG_PATH
 from kleinkram.config import Credentials
+from kleinkram.config import get_config
+from kleinkram.config import save_config
 
 CLI_CALLBACK_ENDPOINT = "/cli/callback"
 OAUTH_SLUG = "/auth/google?state=cli"
@@ -24,16 +25,18 @@ def _has_browser() -> bool:
 
 
 def _headless_auth(*, url: str) -> None:
-    config = Config()
 
-    print(f"Please open the following URL manually to authenticate: {url}")
-    print("Enter the authentication token provided after logging in:")
-    auth_token = getpass("Authentication Token: ")
-    refresh_token = getpass("Refresh Token: ")
+    print(f"please open the following URL manually to authenticate: {url}")
+    print("enter the authentication token provided after logging in:")
+    auth_token = getpass("authentication token: ")
+    refresh_token = getpass("refresh token: ")
 
     if auth_token and refresh_token:
-        creds = Credentials(auth_token=auth_token, refresh_token=refresh_token)
-        config.save_credentials(creds)
+        config = get_config()
+        config.credentials = Credentials(
+            auth_token=auth_token, refresh_token=refresh_token
+        )
+        save_config(config)
         print(f"Authentication complete. Tokens saved to {CONFIG_PATH}.")
     else:
         raise ValueError("Please provided tokens.")
@@ -50,12 +53,12 @@ class OAuthCallbackHandler(BaseHTTPRequestHandler):
                     auth_token=params.get("authtoken")[0],  # type: ignore
                     refresh_token=params.get("refreshtoken")[0],  # type: ignore
                 )
+                config = get_config()
+                config.credentials = creds
+                save_config(config)
             except Exception:
                 raise RuntimeError("Failed to fetch authentication tokens.")
 
-            config = Config()
-            config.save_credentials(creds)
-
             self.send_response(200)
             self.send_header("Content-type", "text/html")
             self.end_headers()
@@ -78,17 +81,15 @@ def _browser_auth(*, url: str) -> None:
 
 
 def login_flow(*, key: Optional[str] = None, headless: bool = False) -> None:
-    config = Config(overwrite=True)
-
+    config = get_config()
     # use cli key login
     if key is not None:
-        creds = Credentials(cli_key=key)
-        config.save_credentials(creds)
-
-    url = f"{config.endpoint}{OAUTH_SLUG}"
+        config.credentials = Credentials(cli_key=key)
+        save_config(config)
+        return
 
+    oauth_url = f"{config.endpoint.api}{OAUTH_SLUG}"
     if not headless and _has_browser():
-        _browser_auth(url=url)
+        _browser_auth(url=oauth_url)
     else:
-        headless_url = f"{url}-no-redirect"
-        _headless_auth(url=headless_url)
+        _headless_auth(url=f"{oauth_url}-no-redirect")

kleinkram/{commands/download.py → cli/_download.py}

@@ -6,17 +6,14 @@ from typing import List
 from typing import Optional
 
 import typer
+
+import kleinkram.core
 from kleinkram.api.client import AuthenticatedClient
-from kleinkram.api.file_transfer import download_files
+from kleinkram.api.query import FileQuery
+from kleinkram.api.query import MissionQuery
+from kleinkram.api.query import ProjectQuery
 from kleinkram.config import get_shared_state
-from kleinkram.models import files_to_table
-from kleinkram.resources import FileSpec
-from kleinkram.resources import get_files_by_spec
-from kleinkram.resources import MissionSpec
-from kleinkram.resources import ProjectSpec
 from kleinkram.utils import split_args
-from rich.console import Console
-
 
 logger = logging.getLogger(__name__)
 
@@ -55,49 +52,26 @@ def download(
         typer.confirm(f"Destination {dest_dir} does not exist. Create it?", abort=True)
     dest_dir.mkdir(parents=True, exist_ok=True)
 
-    # get file spec
+    # get file query
     file_ids, file_patterns = split_args(files or [])
     mission_ids, mission_patterns = split_args(missions or [])
     project_ids, project_patterns = split_args(projects or [])
 
-    project_spec = ProjectSpec(patterns=project_patterns, ids=project_ids)
-    mission_spec = MissionSpec(
+    project_query = ProjectQuery(patterns=project_patterns, ids=project_ids)
+    mission_query = MissionQuery(
         patterns=mission_patterns,
         ids=mission_ids,
-        project_spec=project_spec,
+        project_query=project_query,
     )
-    file_spec = FileSpec(
-        patterns=file_patterns, ids=file_ids, mission_spec=mission_spec
+    file_query = FileQuery(
+        patterns=file_patterns, ids=file_ids, mission_query=mission_query
     )
 
-    client = AuthenticatedClient()
-    parsed_files = get_files_by_spec(client, file_spec)
-
-    if get_shared_state().verbose:
-        table = files_to_table(parsed_files, title="downloading files...")
-        Console().print(table)
-
-    # get paths to files map
-    if (
-        len(set([(file.project_id, file.mission_id) for file in parsed_files])) > 1
-        and not nested
-    ):
-        raise ValueError(
-            "files from multiple missions were selected, consider using `--nested`"
-        )
-    elif not nested:
-        # flat structure
-        paths_to_files = {dest_dir / file.name: file for file in parsed_files}
-    else:
-        # allow for nested directories
-        paths_to_files = {}
-        for file in parsed_files:
-            paths_to_files[
-                dest_dir / file.project_name / file.mission_name / file.name
-            ] = file
-
-    # download files
-    logger.info(f"downloading {paths_to_files} files to {dest_dir}")
-    download_files(
-        client, paths_to_files, verbose=get_shared_state().verbose, overwrite=overwrite
+    kleinkram.core.download(
+        client=AuthenticatedClient(),
+        query=file_query,
+        base_dir=dest_dir,
+        nested=nested,
+        overwrite=overwrite,
+        verbose=get_shared_state().verbose,
     )

kleinkram/cli/_endpoint.py

@@ -0,0 +1,58 @@
+"""
+at the moment the endpoint command lets you specify the api and s3 endpoints
+eventually it will be sufficient to just specify the api endpoint and the s3 endpoint will
+be provided by the api
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import typer
+from rich.console import Console
+
+from kleinkram.config import Endpoint
+from kleinkram.config import add_endpoint
+from kleinkram.config import endpoint_table
+from kleinkram.config import get_config
+from kleinkram.config import select_endpoint
+
+HELP = """\
+Get or set the current endpoint.
+
+The endpoint is used to determine the API server to connect to\
+(default is the API server of https://datasets.leggedrobotics.com).
+"""
+
+endpoint_typer = typer.Typer(
+    name="endpoint",
+    help=HELP,
+    context_settings={"help_option_names": ["-h", "--help"]},
+    invoke_without_command=True,
+)
+
+
+@endpoint_typer.callback()
+def endpoint(
+    name: Optional[str] = typer.Argument(None, help="Name of the endpoint to use"),
+    api: Optional[str] = typer.Argument(None, help="API endpoint to use"),
+    s3: Optional[str] = typer.Argument(None, help="S3 endpoint to use"),
+) -> None:
+    config = get_config()
+    console = Console()
+
+    if not any([name, api, s3]):
+        console.print(endpoint_table(config))
+    elif name is not None and not any([api, s3]):
+        try:
+            select_endpoint(config, name)
+        except ValueError:
+            console.print(f"Endpoint {name} not found.\n", style="red")
+            console.print(endpoint_table(config))
+    elif not (name and api and s3):
+        raise typer.BadParameter(
+            "to add a new endpoint you must specify the api and s3 endpoints"
+        )
+    else:
+        new_endpoint = Endpoint(name, api, s3)
+        add_endpoint(config, new_endpoint)