kleinkram 0.38.1.dev20241120100707__py3-none-any.whl → 0.38.1.dev20241212075157__py3-none-any.whl

kleinkram/commands/upload.py

@@ -3,24 +3,26 @@ from __future__ import annotations
 from pathlib import Path
 from typing import List
 from typing import Optional
-from uuid import UUID
 
 import typer
 from kleinkram.api.client import AuthenticatedClient
 from kleinkram.api.file_transfer import upload_files
-from kleinkram.api.routes import create_mission
-from kleinkram.api.routes import get_mission_by_id
-from kleinkram.api.routes import get_mission_by_spec
-from kleinkram.api.routes import get_project_id_by_name
-from kleinkram.api.routes import get_tags_map
+from kleinkram.api.routes import _create_mission as _create_mission_api
 from kleinkram.config import get_shared_state
-from kleinkram.errors import MissionDoesNotExist
-from kleinkram.models import MissionByName
+from kleinkram.errors import MissionNotFound
+from kleinkram.errors import ProjectNotFound
+from kleinkram.models import Mission
+from kleinkram.resources import check_mission_spec_is_creatable
+from kleinkram.resources import get_missions_by_spec
+from kleinkram.resources import get_projects_by_spec
+from kleinkram.resources import InvalidMissionSpec
+from kleinkram.resources import mission_spec_is_unique
+from kleinkram.resources import MissionSpec
+from kleinkram.resources import ProjectSpec
 from kleinkram.utils import check_file_paths
 from kleinkram.utils import get_filename_map
-from kleinkram.utils import get_valid_mission_spec
 from kleinkram.utils import load_metadata
-from kleinkram.utils import to_name_or_uuid
+from kleinkram.utils import split_args
 from rich.console import Console
 
 
@@ -36,6 +38,45 @@ upload_typer = typer.Typer(
 )
 
 
+# TODO: move this to core
+def _create_mission(
+    client: AuthenticatedClient,
+    mission_spec: MissionSpec,
+    metadata_path: Optional[Path],
+    ignore_missing_tags: bool,
+) -> Mission:
+    check_mission_spec_is_creatable(mission_spec)
+    mission_name = mission_spec.patterns[0]
+
+    # get the metadata
+    metadata_dct = {}
+    if metadata_path is not None:
+        metadata_dct = load_metadata(metadata_path)
+
+    # get project
+    projects = get_projects_by_spec(client, mission_spec.project_spec)
+
+    if not projects:
+        raise ProjectNotFound(f"project {mission_spec.project_spec} does not exist")
+    elif len(projects) > 1:
+        raise AssertionError("unreachable")
+
+    parsed_project = projects[0]
+
+    _create_mission_api(
+        client,
+        parsed_project.id,
+        mission_name,
+        metadata=metadata_dct,
+        ignore_missing_tags=ignore_missing_tags,
+    )
+
+    missions = get_missions_by_spec(client, mission_spec)
+    assert len(missions) is not None, "unreachable, the ghost is back"
+
+    return missions[0]
+
+
 @upload_typer.callback()
 def upload(
     files: List[str] = typer.Argument(help="files to upload"),
@@ -50,21 +91,13 @@ def upload(
     fix_filenames: bool = typer.Option(False, help="fix filenames"),
     ignore_missing_tags: bool = typer.Option(False, help="ignore mission tags"),
 ) -> None:
-    _project = to_name_or_uuid(project) if project else None
-    _mission = to_name_or_uuid(mission)
-
-    client = AuthenticatedClient()
-
     # check files and `fix` filenames
     if files is None:
         files = []
 
     file_paths = [Path(file) for file in files]
     check_file_paths(file_paths)
-
-    files_map = get_filename_map(
-        [Path(file) for file in files],
-    )
+    files_map = get_filename_map(file_paths)
 
     if not fix_filenames:
         for name, path in files_map.items():
@@ -73,51 +106,43 @@ def upload(
                     f"invalid filename format {path.name}, use `--fix-filenames`"
                 )
 
-    # parse the mission spec and get mission
-    mission_spec = get_valid_mission_spec(_mission, _project)
-    mission_parsed = get_mission_by_spec(client, mission_spec)
+    mission_ids, mission_patterns = split_args([mission])
+    project_ids, project_patterns = split_args([project] if project else [])
+
+    project_spec = ProjectSpec(ids=project_ids, patterns=project_patterns)
+    mission_spec = MissionSpec(
+        ids=mission_ids,
+        patterns=mission_patterns,
+        project_spec=project_spec,
+    )
 
-    if not create and mission_parsed is None:
-        raise MissionDoesNotExist(f"mission: {mission} does not exist, use `--create`")
+    if not mission_spec_is_unique(mission_spec):
+        raise InvalidMissionSpec(f"mission spec is not unique: {mission_spec}")
 
-    # create missing mission
-    if mission_parsed is None:
-        if not isinstance(mission_spec, MissionByName):
-            raise ValueError(
-                "cannot create mission using mission id, pecify a mission name"
-            )
+    client = AuthenticatedClient()
+    missions = get_missions_by_spec(client, mission_spec)
 
-        # get the metadata
-        tags_dct = {}
-        if metadata is not None:
-            metadata_dct = load_metadata(Path(metadata))
-            tags_dct = get_tags_map(client, metadata_dct)
+    if len(missions) > 1:
+        raise AssertionError("unreachable")
 
-        # get project id
-        if isinstance(mission_spec.project, UUID):
-            project_id = mission_spec.project
-        else:
-            project_id = get_project_id_by_name(client, mission_spec.project)
-            if project_id is None:
-                raise ValueError(
-                    f"unable to create mission, project: {mission_spec.project} not found"
-                )
+    if not create and not missions:
+        raise MissionNotFound(f"mission: {mission_spec} does not exist, use `--create`")
 
-        mission_id = create_mission(
+    # create mission if it does not exist
+    mission_parsed = (
+        missions[0]
+        if missions
+        else _create_mission(
             client,
-            project_id,
-            mission_spec.name,
-            tags=tags_dct,
+            mission_spec,
+            metadata_path=Path(metadata) if metadata else None,
             ignore_missing_tags=ignore_missing_tags,
         )
+    )
 
-        mission_parsed = get_mission_by_id(client, mission_id)
-        assert mission_parsed is not None, "unreachable"
-
+    console = Console()
     filtered_files_map = {}
     remote_file_names = [file.name for file in mission_parsed.files]
-
-    console = Console()
     for name, path in files_map.items():
         if name in remote_file_names:
             console.print(
@@ -131,6 +156,7 @@ def upload(
         return
 
     upload_files(
+        client,
         filtered_files_map,
         mission_parsed.id,
         n_workers=2,

kleinkram/commands/verify.py

@@ -4,20 +4,26 @@ import logging
 import sys
 from enum import Enum
 from pathlib import Path
+from typing import Dict
 from typing import List
 from typing import Optional
 
 import typer
 from kleinkram.api.client import AuthenticatedClient
-from kleinkram.api.routes import get_mission_by_spec
 from kleinkram.config import get_shared_state
-from kleinkram.errors import MissionDoesNotExist
+from kleinkram.errors import InvalidMissionSpec
+from kleinkram.errors import MissionNotFound
 from kleinkram.models import FileState
+from kleinkram.resources import FileSpec
+from kleinkram.resources import get_files_by_spec
+from kleinkram.resources import get_missions_by_spec
+from kleinkram.resources import mission_spec_is_unique
+from kleinkram.resources import MissionSpec
+from kleinkram.resources import ProjectSpec
 from kleinkram.utils import b64_md5
 from kleinkram.utils import check_file_paths
 from kleinkram.utils import get_filename_map
-from kleinkram.utils import get_valid_mission_spec
-from kleinkram.utils import to_name_or_uuid
+from kleinkram.utils import split_args
 from rich.console import Console
 from rich.table import Table
 from rich.text import Text
@@ -29,8 +35,9 @@ logger = logging.getLogger(__name__)
 class FileVerificationStatus(str, Enum):
     UPLAODED = "uploaded"
     UPLOADING = "uploading"
+    COMPUTING_HASH = "computing hash"
     MISSING = "missing"
-    CORRUPTED = "hash mismatch"
+    MISMATCHED_HASH = "hash mismatch"
     UNKNOWN = "unknown"
 
 
@@ -38,8 +45,9 @@ FILE_STATUS_STYLES = {
     FileVerificationStatus.UPLAODED: "green",
     FileVerificationStatus.UPLOADING: "yellow",
     FileVerificationStatus.MISSING: "yellow",
-    FileVerificationStatus.CORRUPTED: "red",
+    FileVerificationStatus.MISMATCHED_HASH: "red",
     FileVerificationStatus.UNKNOWN: "gray",
+    FileVerificationStatus.COMPUTING_HASH: "purple",
 }
 
 
@@ -59,60 +67,75 @@ def verify(
     mission: str = typer.Option(..., "--mission", "-m", help="mission id or name"),
     skip_hash: bool = typer.Option(False, help="skip hash check"),
 ) -> None:
-    _project = to_name_or_uuid(project) if project else None
-    _mission = to_name_or_uuid(mission)
-
-    client = AuthenticatedClient()
-
+    # get all filepaths
     if files is None:
         files = []
 
-    mission_spec = get_valid_mission_spec(_mission, _project)
-    mission_parsed = get_mission_by_spec(client, mission_spec)
-
-    if mission_parsed is None:
-        raise MissionDoesNotExist(f"Mission {mission} does not exist")
-
-    # check file types
     file_paths = [Path(file) for file in files]
     check_file_paths(file_paths)
+    files_map = get_filename_map(file_paths)
+
+    # get the mission by the provided spec
+    mission_ids, mission_patterns = split_args([mission])
+    project_ids, project_patterns = split_args([project] if project else [])
 
-    filename_map = get_filename_map(file_paths)
-    remote_files = {file.name: file for file in mission_parsed.files}
+    project_spec = ProjectSpec(ids=project_ids, patterns=project_patterns)
+    mission_spec = MissionSpec(
+        ids=mission_ids, patterns=mission_patterns, project_spec=project_spec
+    )
+    file_spec = FileSpec(mission_spec=mission_spec)
 
-    status_dct = {}
+    client = AuthenticatedClient()
+
+    # check first that the mission even exists, the mission could be empty
+    if not mission_spec_is_unique(mission_spec):
+        raise InvalidMissionSpec(f"mission spec is not unique: {mission_spec}")
+    missions = get_missions_by_spec(client, mission_spec)
+    if len(missions) > 1:
+        raise AssertionError("unreachable")
+    if not missions:
+        raise MissionNotFound(f"mission: {mission_spec} does not exist")
+
+    # get all files from that mission
+    remote_files = {file.name: file for file in get_files_by_spec(client, file_spec)}
+
+    # verify files
+    file_status: Dict[Path, FileVerificationStatus] = {}
     for name, file in tqdm(
-        filename_map.items(),
+        files_map.items(),
         desc="verifying files",
         unit="file",
-        disable=skip_hash or not get_shared_state().verbose,
+        disable=not get_shared_state().verbose,
     ):
         if name not in remote_files:
-            status_dct[file] = FileVerificationStatus.MISSING
+            file_status[file] = FileVerificationStatus.MISSING
             continue
 
-        state = remote_files[name].state
-
-        if state == FileState.UPLOADING:
-            status_dct[file] = FileVerificationStatus.UPLOADING
-        elif state == FileState.OK and remote_files[name].hash != b64_md5(file):
-            status_dct[file] = FileVerificationStatus.CORRUPTED
-        elif state == FileState.OK:
-            status_dct[file] = FileVerificationStatus.UPLAODED
+        remote_file = remote_files[name]
+
+        if remote_file.state == FileState.UPLOADING:
+            file_status[file] = FileVerificationStatus.UPLOADING
+        elif remote_file.state == FileState.OK:
+            if remote_file.hash is None:
+                file_status[file] = FileVerificationStatus.COMPUTING_HASH
+            elif skip_hash or remote_file.hash == b64_md5(file):
+                file_status[file] = FileVerificationStatus.UPLAODED
+            else:
+                file_status[file] = FileVerificationStatus.MISMATCHED_HASH
         else:
-            status_dct[file] = FileVerificationStatus.UNKNOWN
+            file_status[file] = FileVerificationStatus.UNKNOWN
 
     if get_shared_state().verbose:
         table = Table(title="file status")
         table.add_column("filename", style="cyan")
         table.add_column("status", style="green")
 
-        for path, status in status_dct.items():
+        for path, status in file_status.items():
             table.add_row(str(path), Text(status, style=FILE_STATUS_STYLES[status]))
 
         Console().print(table)
     else:
-        for path, status in status_dct.items():
+        for path, status in file_status.items():
             stream = (
                 sys.stdout if status == FileVerificationStatus.UPLAODED else sys.stderr
             )

kleinkram/config.py

@@ -12,7 +12,6 @@ from typing import Optional
 
 from kleinkram._version import __local__
 from kleinkram._version import __version__
-from kleinkram.errors import CorruptedConfigFile
 from kleinkram.errors import InvalidConfigFile
 
 CONFIG_PATH = Path().home() / ".kleinkram.json"
@@ -71,7 +70,7 @@ class Config:
 
         try:
             self._read_config()
-        except (InvalidConfigFile, CorruptedConfigFile):
+        except InvalidConfigFile:
             if not overwrite:
                 self.credentials = {}
                 self.endpoint = default_endpoint
@@ -84,7 +83,7 @@ class Config:
             try:
                 content = json.load(file)
             except Exception:
-                raise CorruptedConfigFile
+                raise InvalidConfigFile
 
         endpoint = content.get(JSON_ENDPOINT_KEY, None)
         if not isinstance(endpoint, str):

kleinkram/errors.py

@@ -1,65 +1,82 @@
 from __future__ import annotations
 
-LOGIN_MESSAGE = "Please login using `klein login`."
+from typing import Any
+from typing import Callable
+from typing import OrderedDict
+from typing import Type
 
+import typer
 
-class InvalidMissionSpec(Exception): ...
+LOGIN_MESSAGE = "Please login using `klein login`."
+INVALID_CONFIG_MESSAGE = "Invalid config file."
 
 
-class InvalidFileSpec(Exception): ...
+class ParsingError(Exception): ...
 
 
-class MissionExists(Exception): ...
+class InvalidMissionSpec(Exception): ...
 
 
-class MissionDoesNotExist(Exception): ...
+class InvalidProjectSpec(Exception): ...
 
 
-class NoPermission(Exception): ...
+class MissionExists(Exception): ...
 
 
-class UploadCredentialsFailed(Exception): ...
+class MissionNotFound(Exception): ...
 
 
-class AccessDeniedException(Exception):
-    def __init__(self, message: str, api_error: str):
-        self.message = message
-        self.api_error = api_error
+class ProjectNotFound(Exception): ...
 
 
-class NotAuthenticatedException(Exception):
-    def __init__(self, endpoint: str):
-        message = (
-            f"You are not authenticated on endpoint '{endpoint}'.\n{LOGIN_MESSAGE}"
-        )
-        super().__init__(message)
+class AccessDenied(Exception): ...
 
 
-class CorruptedFile(Exception): ...
+class NotAuthenticated(Exception):
+    def __init__(self) -> None:
+        super().__init__(LOGIN_MESSAGE)
 
 
-class NameIsValidUUID(Exception): ...
+class InvalidCLIVersion(Exception): ...
 
 
-class NotValidUUID(Exception): ...
+class FileTypeNotSupported(Exception): ...
 
 
-class UploadFailed(Exception): ...
+class InvalidConfigFile(Exception):
+    def __init__(self) -> None:
+        super().__init__(INVALID_CONFIG_MESSAGE)
 
 
-class InvalidCLIVersion(Exception): ...
+ExceptionHandler = Callable[[Exception], int]
 
 
-class FileTypeNotSupported(Exception): ...
+class ErrorHandledTyper(typer.Typer):
+    """\
+    error handlers that are last added will be used first
+    """
 
+    _error_handlers: OrderedDict[Type[Exception], ExceptionHandler]
 
-class InvalidConfigFile(Exception):
-    def __init__(self) -> None:
-        super().__init__("Invalid config file.")
+    def error_handler(
+        self, exc: type[Exception]
+    ) -> Callable[[ExceptionHandler], ExceptionHandler]:
+        def dec(func: ExceptionHandler) -> ExceptionHandler:
+            self._error_handlers[exc] = func
+            return func
 
+        return dec
 
-class CorruptedConfigFile(Exception):
-    def __init__(self) -> None:
-        super().__init__(
-            "Config file is corrupted.\nPlease run `klein login` to re-authenticate."
-        )
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._error_handlers = OrderedDict()
+
+    def __call__(self, *args: Any, **kwargs: Any) -> int:
+        try:
+            return super().__call__(*args, **kwargs)
+        except Exception as e:
+            for tp, handler in reversed(self._error_handlers.items()):
+                if isinstance(e, tp):
+                    exit_code = handler(e)
+                    raise SystemExit(exit_code)
+            raise

kleinkram/models.py

@@ -14,7 +14,7 @@ from rich.table import Table
 from rich.text import Text
 
 
-@dataclass(frozen=True, eq=True)
+@dataclass(eq=True)
 class Project:
     id: UUID
     name: str
@@ -22,7 +22,7 @@ class Project:
     missions: List[Mission] = field(default_factory=list)
 
 
-@dataclass(frozen=True, eq=True)
+@dataclass(eq=True)
 class Mission:
     id: UUID
     name: str

kleinkram/resources.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from dataclasses import field
+from itertools import chain
+from typing import List
+from uuid import UUID
+
+from kleinkram.api.client import AuthenticatedClient
+from kleinkram.api.routes import _get_files_by_mission
+from kleinkram.api.routes import _get_missions_by_project
+from kleinkram.api.routes import _get_projects
+from kleinkram.errors import InvalidMissionSpec
+from kleinkram.errors import InvalidProjectSpec
+from kleinkram.models import File
+from kleinkram.models import Mission
+from kleinkram.models import Project
+from kleinkram.utils import filtered_by_patterns
+
+MAX_PARALLEL_REQUESTS = 32
+SPECIAL_PATTERN_CHARS = ["*", "?", "[", "]"]
+
+
+@dataclass
+class ProjectSpec:
+    patterns: List[str] = field(default_factory=list)
+    ids: List[UUID] = field(default_factory=list)
+
+
+@dataclass
+class MissionSpec:
+    patterns: List[str] = field(default_factory=list)
+    ids: List[UUID] = field(default_factory=list)
+    project_spec: ProjectSpec = field(default=ProjectSpec())
+
+
+@dataclass
+class FileSpec:
+    patterns: List[str] = field(default_factory=list)
+    ids: List[UUID] = field(default_factory=list)
+    mission_spec: MissionSpec = field(default=MissionSpec())
+
+
+def check_mission_spec_is_creatable(spec: MissionSpec) -> None:
+    if not mission_spec_is_unique(spec):
+        raise InvalidMissionSpec(f"Mission spec is not unique: {spec}")
+    # cant create a missing by id
+    if spec.ids:
+        raise InvalidMissionSpec(f"cant create mission by id: {spec}")
+
+
+def check_project_spec_is_creatable(spec: ProjectSpec) -> None:
+    if not project_spec_is_unique(spec):
+        raise InvalidProjectSpec(f"Project spec is not unique: {spec}")
+    # cant create a missing by id
+    if spec.ids:
+        raise InvalidProjectSpec(f"cant create project by id: {spec}")
+
+
+def _pattern_is_unique(pattern: str) -> bool:
+    for char in SPECIAL_PATTERN_CHARS:
+        if char in pattern:
+            return False
+    return True
+
+
+def project_spec_is_unique(spec: ProjectSpec) -> bool:
+    # a single project id is specified
+    if len(spec.ids) == 1 and not spec.patterns:
+        return True
+
+    # a single project name is specified
+    if len(spec.patterns) == 1 and _pattern_is_unique(spec.patterns[0]):
+        return True
+    return False
+
+
+def mission_spec_is_unique(spec: MissionSpec) -> bool:
+    # a single mission id is specified
+    if len(spec.ids) == 1 and not spec.patterns:
+        return True
+
+    # a single mission name a unique project spec are specified
+    if (
+        project_spec_is_unique(spec.project_spec)
+        and len(spec.patterns) == 1
+        and _pattern_is_unique(spec.patterns[0])
+    ):
+        return True
+    return False
+
+
+def get_projects_by_spec(
+    client: AuthenticatedClient, spec: ProjectSpec
+) -> List[Project]:
+    projects = _get_projects(client)
+
+    matched_names = filtered_by_patterns(
+        [project.name for project in projects], spec.patterns
+    )
+
+    if not spec.patterns and not spec.ids:
+        return projects
+
+    return [
+        project
+        for project in projects
+        if project.name in matched_names or project.id in spec.ids
+    ]
+
+
+def get_missions_by_spec(
+    client: AuthenticatedClient, spec: MissionSpec
+) -> List[Mission]:
+    projects = get_projects_by_spec(client, spec.project_spec)
+
+    with ThreadPoolExecutor(max_workers=MAX_PARALLEL_REQUESTS) as executor:
+        missions = chain.from_iterable(
+            executor.map(
+                lambda project: _get_missions_by_project(client, project), projects
+            )
+        )
+
+    missions = list(missions)
+
+    if not spec.patterns and not spec.ids:
+        return list(missions)
+
+    matched_names = filtered_by_patterns(
+        [mission.name for mission in missions], spec.patterns
+    )
+
+    filter = [
+        mission
+        for mission in missions
+        if mission.name in matched_names or mission.id in spec.patterns
+    ]
+
+    return filter
+
+
+def get_files_by_spec(client: AuthenticatedClient, spec: FileSpec) -> List[File]:
+    missions = get_missions_by_spec(client, spec.mission_spec)
+
+    # collect files
+    with ThreadPoolExecutor(max_workers=MAX_PARALLEL_REQUESTS) as executor:
+        files = chain.from_iterable(
+            executor.map(
+                lambda mission: _get_files_by_mission(client, mission), missions
+            )
+        )
+
+    if not spec.patterns and not spec.ids:
+        return list(files)
+    matched_names = filtered_by_patterns([file.name for file in files], spec.patterns)
+
+    return [file for file in files if file.name in matched_names or file.id in spec.ids]