kleinkram 0.38.1.dev20241212075157__py3-none-any.whl → 0.38.1.dev20250113080249__py3-none-any.whl

kleinkram/commands/upload.py

@@ -1,164 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-from typing import List
-from typing import Optional
-
-import typer
-from kleinkram.api.client import AuthenticatedClient
-from kleinkram.api.file_transfer import upload_files
-from kleinkram.api.routes import _create_mission as _create_mission_api
-from kleinkram.config import get_shared_state
-from kleinkram.errors import MissionNotFound
-from kleinkram.errors import ProjectNotFound
-from kleinkram.models import Mission
-from kleinkram.resources import check_mission_spec_is_creatable
-from kleinkram.resources import get_missions_by_spec
-from kleinkram.resources import get_projects_by_spec
-from kleinkram.resources import InvalidMissionSpec
-from kleinkram.resources import mission_spec_is_unique
-from kleinkram.resources import MissionSpec
-from kleinkram.resources import ProjectSpec
-from kleinkram.utils import check_file_paths
-from kleinkram.utils import get_filename_map
-from kleinkram.utils import load_metadata
-from kleinkram.utils import split_args
-from rich.console import Console
-
-
-HELP = """\
-Upload files to kleinkram.
-"""
-
-upload_typer = typer.Typer(
-    name="upload",
-    no_args_is_help=True,
-    invoke_without_command=True,
-    help=HELP,
-)
-
-
-# TODO: move this to core
-def _create_mission(
-    client: AuthenticatedClient,
-    mission_spec: MissionSpec,
-    metadata_path: Optional[Path],
-    ignore_missing_tags: bool,
-) -> Mission:
-    check_mission_spec_is_creatable(mission_spec)
-    mission_name = mission_spec.patterns[0]
-
-    # get the metadata
-    metadata_dct = {}
-    if metadata_path is not None:
-        metadata_dct = load_metadata(metadata_path)
-
-    # get project
-    projects = get_projects_by_spec(client, mission_spec.project_spec)
-
-    if not projects:
-        raise ProjectNotFound(f"project {mission_spec.project_spec} does not exist")
-    elif len(projects) > 1:
-        raise AssertionError("unreachable")
-
-    parsed_project = projects[0]
-
-    _create_mission_api(
-        client,
-        parsed_project.id,
-        mission_name,
-        metadata=metadata_dct,
-        ignore_missing_tags=ignore_missing_tags,
-    )
-
-    missions = get_missions_by_spec(client, mission_spec)
-    assert len(missions) is not None, "unreachable, the ghost is back"
-
-    return missions[0]
-
-
-@upload_typer.callback()
-def upload(
-    files: List[str] = typer.Argument(help="files to upload"),
-    project: Optional[str] = typer.Option(
-        None, "--project", "-p", help="project id or name"
-    ),
-    mission: str = typer.Option(..., "--mission", "-m", help="mission id or name"),
-    create: bool = typer.Option(False, help="create mission if it does not exist"),
-    metadata: Optional[str] = typer.Option(
-        None, help="path to metadata file (json or yaml)"
-    ),
-    fix_filenames: bool = typer.Option(False, help="fix filenames"),
-    ignore_missing_tags: bool = typer.Option(False, help="ignore mission tags"),
-) -> None:
-    # check files and `fix` filenames
-    if files is None:
-        files = []
-
-    file_paths = [Path(file) for file in files]
-    check_file_paths(file_paths)
-    files_map = get_filename_map(file_paths)
-
-    if not fix_filenames:
-        for name, path in files_map.items():
-            if name != path.name:
-                raise ValueError(
-                    f"invalid filename format {path.name}, use `--fix-filenames`"
-                )
-
-    mission_ids, mission_patterns = split_args([mission])
-    project_ids, project_patterns = split_args([project] if project else [])
-
-    project_spec = ProjectSpec(ids=project_ids, patterns=project_patterns)
-    mission_spec = MissionSpec(
-        ids=mission_ids,
-        patterns=mission_patterns,
-        project_spec=project_spec,
-    )
-
-    if not mission_spec_is_unique(mission_spec):
-        raise InvalidMissionSpec(f"mission spec is not unique: {mission_spec}")
-
-    client = AuthenticatedClient()
-    missions = get_missions_by_spec(client, mission_spec)
-
-    if len(missions) > 1:
-        raise AssertionError("unreachable")
-
-    if not create and not missions:
-        raise MissionNotFound(f"mission: {mission_spec} does not exist, use `--create`")
-
-    # create mission if it does not exist
-    mission_parsed = (
-        missions[0]
-        if missions
-        else _create_mission(
-            client,
-            mission_spec,
-            metadata_path=Path(metadata) if metadata else None,
-            ignore_missing_tags=ignore_missing_tags,
-        )
-    )
-
-    console = Console()
-    filtered_files_map = {}
-    remote_file_names = [file.name for file in mission_parsed.files]
-    for name, path in files_map.items():
-        if name in remote_file_names:
-            console.print(
-                f"file: {name} (path: {path}) already exists in mission", style="yellow"
-            )
-        else:
-            filtered_files_map[name] = path
-
-    if not filtered_files_map:
-        console.print("\nNO FILES UPLOADED", style="yellow")
-        return
-
-    upload_files(
-        client,
-        filtered_files_map,
-        mission_parsed.id,
-        n_workers=2,
-        verbose=get_shared_state().verbose,
-    )

kleinkram/commands/verify.py

@@ -1,142 +0,0 @@
-from __future__ import annotations
-
-import logging
-import sys
-from enum import Enum
-from pathlib import Path
-from typing import Dict
-from typing import List
-from typing import Optional
-
-import typer
-from kleinkram.api.client import AuthenticatedClient
-from kleinkram.config import get_shared_state
-from kleinkram.errors import InvalidMissionSpec
-from kleinkram.errors import MissionNotFound
-from kleinkram.models import FileState
-from kleinkram.resources import FileSpec
-from kleinkram.resources import get_files_by_spec
-from kleinkram.resources import get_missions_by_spec
-from kleinkram.resources import mission_spec_is_unique
-from kleinkram.resources import MissionSpec
-from kleinkram.resources import ProjectSpec
-from kleinkram.utils import b64_md5
-from kleinkram.utils import check_file_paths
-from kleinkram.utils import get_filename_map
-from kleinkram.utils import split_args
-from rich.console import Console
-from rich.table import Table
-from rich.text import Text
-from tqdm import tqdm
-
-logger = logging.getLogger(__name__)
-
-
-class FileVerificationStatus(str, Enum):
-    UPLAODED = "uploaded"
-    UPLOADING = "uploading"
-    COMPUTING_HASH = "computing hash"
-    MISSING = "missing"
-    MISMATCHED_HASH = "hash mismatch"
-    UNKNOWN = "unknown"
-
-
-FILE_STATUS_STYLES = {
-    FileVerificationStatus.UPLAODED: "green",
-    FileVerificationStatus.UPLOADING: "yellow",
-    FileVerificationStatus.MISSING: "yellow",
-    FileVerificationStatus.MISMATCHED_HASH: "red",
-    FileVerificationStatus.UNKNOWN: "gray",
-    FileVerificationStatus.COMPUTING_HASH: "purple",
-}
-
-
-HELP = """\
-Verify if files were uploaded correctly.
-"""
-
-verify_typer = typer.Typer(name="verify", invoke_without_command=True, help=HELP)
-
-
-@verify_typer.callback()
-def verify(
-    files: List[str] = typer.Argument(help="files to upload"),
-    project: Optional[str] = typer.Option(
-        None, "--project", "-p", help="project id or name"
-    ),
-    mission: str = typer.Option(..., "--mission", "-m", help="mission id or name"),
-    skip_hash: bool = typer.Option(False, help="skip hash check"),
-) -> None:
-    # get all filepaths
-    if files is None:
-        files = []
-
-    file_paths = [Path(file) for file in files]
-    check_file_paths(file_paths)
-    files_map = get_filename_map(file_paths)
-
-    # get the mission by the provided spec
-    mission_ids, mission_patterns = split_args([mission])
-    project_ids, project_patterns = split_args([project] if project else [])
-
-    project_spec = ProjectSpec(ids=project_ids, patterns=project_patterns)
-    mission_spec = MissionSpec(
-        ids=mission_ids, patterns=mission_patterns, project_spec=project_spec
-    )
-    file_spec = FileSpec(mission_spec=mission_spec)
-
-    client = AuthenticatedClient()
-
-    # check first that the mission even exists, the mission could be empty
-    if not mission_spec_is_unique(mission_spec):
-        raise InvalidMissionSpec(f"mission spec is not unique: {mission_spec}")
-    missions = get_missions_by_spec(client, mission_spec)
-    if len(missions) > 1:
-        raise AssertionError("unreachable")
-    if not missions:
-        raise MissionNotFound(f"mission: {mission_spec} does not exist")
-
-    # get all files from that mission
-    remote_files = {file.name: file for file in get_files_by_spec(client, file_spec)}
-
-    # verify files
-    file_status: Dict[Path, FileVerificationStatus] = {}
-    for name, file in tqdm(
-        files_map.items(),
-        desc="verifying files",
-        unit="file",
-        disable=not get_shared_state().verbose,
-    ):
-        if name not in remote_files:
-            file_status[file] = FileVerificationStatus.MISSING
-            continue
-
-        remote_file = remote_files[name]
-
-        if remote_file.state == FileState.UPLOADING:
-            file_status[file] = FileVerificationStatus.UPLOADING
-        elif remote_file.state == FileState.OK:
-            if remote_file.hash is None:
-                file_status[file] = FileVerificationStatus.COMPUTING_HASH
-            elif skip_hash or remote_file.hash == b64_md5(file):
-                file_status[file] = FileVerificationStatus.UPLAODED
-            else:
-                file_status[file] = FileVerificationStatus.MISMATCHED_HASH
-        else:
-            file_status[file] = FileVerificationStatus.UNKNOWN
-
-    if get_shared_state().verbose:
-        table = Table(title="file status")
-        table.add_column("filename", style="cyan")
-        table.add_column("status", style="green")
-
-        for path, status in file_status.items():
-            table.add_row(str(path), Text(status, style=FILE_STATUS_STYLES[status]))
-
-        Console().print(table)
-    else:
-        for path, status in file_status.items():
-            stream = (
-                sys.stdout if status == FileVerificationStatus.UPLAODED else sys.stderr
-            )
-            print(path, file=stream)

kleinkram/consts.py

@@ -1,8 +0,0 @@
-from __future__ import annotations
-
-LOCAL_API_URL = "http://localhost:3000"
-LOCAL_S3_URL = "http://localhost:9000"
-
-
-API_URL = "http://localhost:3000"
-# API_URL = "https://api.datasets.leggedrobotics.com"

kleinkram/enums.py

@@ -1,10 +0,0 @@
-from __future__ import annotations
-
-from enum import Enum
-
-
-class PermissionLevel(Enum):
-    READ = 0
-    CREATE = 10
-    WRITE = 20
-    DELETE = 30

kleinkram/resources.py

@@ -1,158 +0,0 @@
-from __future__ import annotations
-
-from concurrent.futures import ThreadPoolExecutor
-from dataclasses import dataclass
-from dataclasses import field
-from itertools import chain
-from typing import List
-from uuid import UUID
-
-from kleinkram.api.client import AuthenticatedClient
-from kleinkram.api.routes import _get_files_by_mission
-from kleinkram.api.routes import _get_missions_by_project
-from kleinkram.api.routes import _get_projects
-from kleinkram.errors import InvalidMissionSpec
-from kleinkram.errors import InvalidProjectSpec
-from kleinkram.models import File
-from kleinkram.models import Mission
-from kleinkram.models import Project
-from kleinkram.utils import filtered_by_patterns
-
-MAX_PARALLEL_REQUESTS = 32
-SPECIAL_PATTERN_CHARS = ["*", "?", "[", "]"]
-
-
-@dataclass
-class ProjectSpec:
-    patterns: List[str] = field(default_factory=list)
-    ids: List[UUID] = field(default_factory=list)
-
-
-@dataclass
-class MissionSpec:
-    patterns: List[str] = field(default_factory=list)
-    ids: List[UUID] = field(default_factory=list)
-    project_spec: ProjectSpec = field(default=ProjectSpec())
-
-
-@dataclass
-class FileSpec:
-    patterns: List[str] = field(default_factory=list)
-    ids: List[UUID] = field(default_factory=list)
-    mission_spec: MissionSpec = field(default=MissionSpec())
-
-
-def check_mission_spec_is_creatable(spec: MissionSpec) -> None:
-    if not mission_spec_is_unique(spec):
-        raise InvalidMissionSpec(f"Mission spec is not unique: {spec}")
-    # cant create a missing by id
-    if spec.ids:
-        raise InvalidMissionSpec(f"cant create mission by id: {spec}")
-
-
-def check_project_spec_is_creatable(spec: ProjectSpec) -> None:
-    if not project_spec_is_unique(spec):
-        raise InvalidProjectSpec(f"Project spec is not unique: {spec}")
-    # cant create a missing by id
-    if spec.ids:
-        raise InvalidProjectSpec(f"cant create project by id: {spec}")
-
-
-def _pattern_is_unique(pattern: str) -> bool:
-    for char in SPECIAL_PATTERN_CHARS:
-        if char in pattern:
-            return False
-    return True
-
-
-def project_spec_is_unique(spec: ProjectSpec) -> bool:
-    # a single project id is specified
-    if len(spec.ids) == 1 and not spec.patterns:
-        return True
-
-    # a single project name is specified
-    if len(spec.patterns) == 1 and _pattern_is_unique(spec.patterns[0]):
-        return True
-    return False
-
-
-def mission_spec_is_unique(spec: MissionSpec) -> bool:
-    # a single mission id is specified
-    if len(spec.ids) == 1 and not spec.patterns:
-        return True
-
-    # a single mission name a unique project spec are specified
-    if (
-        project_spec_is_unique(spec.project_spec)
-        and len(spec.patterns) == 1
-        and _pattern_is_unique(spec.patterns[0])
-    ):
-        return True
-    return False
-
-
-def get_projects_by_spec(
-    client: AuthenticatedClient, spec: ProjectSpec
-) -> List[Project]:
-    projects = _get_projects(client)
-
-    matched_names = filtered_by_patterns(
-        [project.name for project in projects], spec.patterns
-    )
-
-    if not spec.patterns and not spec.ids:
-        return projects
-
-    return [
-        project
-        for project in projects
-        if project.name in matched_names or project.id in spec.ids
-    ]
-
-
-def get_missions_by_spec(
-    client: AuthenticatedClient, spec: MissionSpec
-) -> List[Mission]:
-    projects = get_projects_by_spec(client, spec.project_spec)
-
-    with ThreadPoolExecutor(max_workers=MAX_PARALLEL_REQUESTS) as executor:
-        missions = chain.from_iterable(
-            executor.map(
-                lambda project: _get_missions_by_project(client, project), projects
-            )
-        )
-
-    missions = list(missions)
-
-    if not spec.patterns and not spec.ids:
-        return list(missions)
-
-    matched_names = filtered_by_patterns(
-        [mission.name for mission in missions], spec.patterns
-    )
-
-    filter = [
-        mission
-        for mission in missions
-        if mission.name in matched_names or mission.id in spec.patterns
-    ]
-
-    return filter
-
-
-def get_files_by_spec(client: AuthenticatedClient, spec: FileSpec) -> List[File]:
-    missions = get_missions_by_spec(client, spec.mission_spec)
-
-    # collect files
-    with ThreadPoolExecutor(max_workers=MAX_PARALLEL_REQUESTS) as executor:
-        files = chain.from_iterable(
-            executor.map(
-                lambda mission: _get_files_by_mission(client, mission), missions
-            )
-        )
-
-    if not spec.patterns and not spec.ids:
-        return list(files)
-    matched_names = filtered_by_patterns([file.name for file in files], spec.patterns)
-
-    return [file for file in files if file.name in matched_names or file.id in spec.ids]