kleinkram 0.38.1.dev20241212075157__tar.gz → 0.38.1.dev20250207122632__tar.gz

{kleinkram-0.38.1.dev20241212075157 → kleinkram-0.38.1.dev20250207122632}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: kleinkram
-Version: 0.38.1.dev20241212075157
+Version: 0.38.1.dev20250207122632
 Summary: give me your bags
 Author: Cyrill Püntener, Dominique Garmier, Johann Schwabe
 Classifier: Programming Language :: Python :: 3
@@ -13,10 +13,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: boto3
 Requires-Dist: botocore
 Requires-Dist: httpx
+Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
 Requires-Dist: rich
 Requires-Dist: tqdm

kleinkram-0.38.1.dev20250207122632/kleinkram/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from kleinkram._version import __version__
+from kleinkram.wrappers import create_mission
+from kleinkram.wrappers import create_project
+from kleinkram.wrappers import delete_file
+from kleinkram.wrappers import delete_files
+from kleinkram.wrappers import delete_mission
+from kleinkram.wrappers import delete_project
+from kleinkram.wrappers import download
+from kleinkram.wrappers import list_files
+from kleinkram.wrappers import list_missions
+from kleinkram.wrappers import list_projects
+from kleinkram.wrappers import update_file
+from kleinkram.wrappers import update_mission
+from kleinkram.wrappers import update_project
+from kleinkram.wrappers import upload
+from kleinkram.wrappers import verify
+
+__all__ = [
+    "__version__",
+    "upload",
+    "verify",
+    "download",
+    "list_files",
+    "list_missions",
+    "list_projects",
+    "update_file",
+    "update_mission",
+    "update_project",
+    "delete_files",
+    "delete_file",
+    "delete_mission",
+    "delete_project",
+    "create_mission",
+    "create_project",
+]

{kleinkram-0.38.1.dev20241212075157 → kleinkram-0.38.1.dev20250207122632}/kleinkram/api/client.py

@@ -5,8 +5,11 @@ from threading import Lock
 from typing import Any
 
 import httpx
-from kleinkram.auth import Config
+
+from kleinkram.config import Config
 from kleinkram.config import Credentials
+from kleinkram.config import get_config
+from kleinkram.config import save_config
 from kleinkram.errors import NotAuthenticated
 
 logger = logging.getLogger(__name__)
@@ -27,27 +30,28 @@ class AuthenticatedClient(httpx.Client):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
 
-        self._config = Config()
+        self._config = get_config()
         self._config_lock = Lock()
 
-        if self._config.has_cli_key:
-            assert self._config.cli_key, "unreachable"
-            logger.info("using cli key...")
-            self.cookies.set(COOKIE_CLI_KEY, self._config.cli_key)
-
-        elif self._config.has_refresh_token:
-            logger.info("using refresh token...")
-            assert self._config.auth_token is not None, "unreachable"
-            self.cookies.set(COOKIE_AUTH_TOKEN, self._config.auth_token)
-        else:
+        if self._config.credentials is None:
             logger.info("not authenticated...")
             raise NotAuthenticated
+        elif (cli_key := self._config.credentials.cli_key) is not None:
+            logger.info("using cli key...")
+            self.cookies.set(COOKIE_CLI_KEY, cli_key)
+        else:
+            logger.info("using refresh token...")
+            assert self._config.credentials.auth_token is not None, "unreachable"
+            self.cookies.set(COOKIE_AUTH_TOKEN, self._config.credentials.auth_token)
 
     def _refresh_token(self) -> None:
-        if self._config.has_cli_key:
+        if self._config.credentials is None:
+            raise NotAuthenticated
+
+        if self._config.credentials.cli_key is not None:
             raise RuntimeError("cannot refresh token when using cli key auth")
 
-        refresh_token = self._config.refresh_token
+        refresh_token = self._config.credentials.refresh_token
         if refresh_token is None:
             raise RuntimeError("no refresh token found")
         self.cookies.set(COOKIE_REFRESH_TOKEN, refresh_token)
@@ -63,7 +67,8 @@ class AuthenticatedClient(httpx.Client):
         logger.info("saving new tokens...")
 
         with self._config_lock:
-            self._config.save_credentials(creds)
+            self._config.credentials = creds
+            save_config(self._config)
 
         self.cookies.set(COOKIE_AUTH_TOKEN, new_access_token)
 
@@ -76,7 +81,7 @@ class AuthenticatedClient(httpx.Client):
             url = f"/{url}"
 
         # try to do a request
-        full_url = f"{self._config.endpoint}{url}"
+        full_url = f"{self._config.endpoint.api}{url}"
         logger.info(f"requesting {method} {full_url}")
         response = super().request(method, full_url, *args, **kwargs)
 

kleinkram-0.38.1.dev20250207122632/kleinkram/api/deser.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+from typing import Any
+from typing import Dict
+from typing import Literal
+from typing import NewType
+from typing import Tuple
+from uuid import UUID
+
+import dateutil.parser
+
+from kleinkram.errors import ParsingError
+from kleinkram.models import File
+from kleinkram.models import FileState
+from kleinkram.models import Mission
+from kleinkram.models import Project
+
+__all__ = [
+    "_parse_project",
+    "_parse_mission",
+    "_parse_file",
+]
+
+
+ProjectObject = NewType("ProjectObject", Dict[str, Any])
+MissionObject = NewType("MissionObject", Dict[str, Any])
+FileObject = NewType("FileObject", Dict[str, Any])
+
+PROJECT_OBJECT_KEYS = []
+MISSION_OBJECT_KEYS = []
+
+MISSION = "mission"
+PROJECT = "project"
+
+
+class FileObjectKeys(str, Enum):
+    UUID = "uuid"
+    FILENAME = "filename"
+    DATE = "date"  # at some point this will become a metadata
+    CREATED_AT = "createdAt"
+    UPDATED_AT = "updatedAt"
+    STATE = "state"
+    SIZE = "size"
+    HASH = "hash"
+    TYPE = "type"
+    CATEGORIES = "categories"
+
+
+class MissionObjectKeys(str, Enum):
+    UUID = "uuid"
+    NAME = "name"
+    DESCRIPTION = "description"
+    CREATED_AT = "createdAt"
+    UPDATED_AT = "updatedAt"
+
+
+class ProjectObjectKeys(str, Enum):
+    UUID = "uuid"
+    NAME = "name"
+    DESCRIPTION = "description"
+    CREATED_AT = "createdAt"
+    UPDATED_AT = "updatedAt"
+
+
+def _get_nested_info(data, key: Literal["mission", "project"]) -> Tuple[UUID, str]:
+    nested_data = data[key]
+    return (
+        UUID(nested_data[ProjectObjectKeys.UUID], version=4),
+        nested_data[ProjectObjectKeys.NAME],
+    )
+
+
+def _parse_datetime(date: str) -> datetime:
+    try:
+        return dateutil.parser.isoparse(date)
+    except ValueError as e:
+        raise ParsingError(f"error parsing date: {date}") from e
+
+
+def _parse_file_state(state: str) -> FileState:
+    try:
+        return FileState(state)
+    except ValueError as e:
+        raise ParsingError(f"error parsing file state: {state}") from e
+
+
+def _parse_project(project_object: ProjectObject) -> Project:
+    try:
+        id_ = UUID(project_object[ProjectObjectKeys.UUID], version=4)
+        name = project_object[ProjectObjectKeys.NAME]
+        description = project_object[ProjectObjectKeys.DESCRIPTION]
+        created_at = _parse_datetime(project_object[ProjectObjectKeys.CREATED_AT])
+        updated_at = _parse_datetime(project_object[ProjectObjectKeys.UPDATED_AT])
+    except Exception as e:
+        raise ParsingError(f"error parsing project: {project_object}") from e
+    return Project(
+        id=id_,
+        name=name,
+        description=description,
+        created_at=created_at,
+        updated_at=updated_at,
+    )
+
+
+def _parse_mission(mission: MissionObject) -> Mission:
+    try:
+        id_ = UUID(mission[MissionObjectKeys.UUID], version=4)
+        name = mission[MissionObjectKeys.NAME]
+        created_at = _parse_datetime(mission[MissionObjectKeys.CREATED_AT])
+        updated_at = _parse_datetime(mission[MissionObjectKeys.UPDATED_AT])
+        metadata = {}  # TODO: this crap is really bad to parse
+
+        project_id, project_name = _get_nested_info(mission, PROJECT)
+
+        parsed = Mission(
+            id=id_,
+            name=name,
+            created_at=created_at,
+            updated_at=updated_at,
+            metadata=metadata,
+            project_id=project_id,
+            project_name=project_name,
+        )
+    except Exception as e:
+        raise ParsingError(f"error parsing mission: {mission}") from e
+    return parsed
+
+
+def _parse_file(file: FileObject) -> File:
+    try:
+        name = file[FileObjectKeys.FILENAME]
+        id_ = UUID(file[FileObjectKeys.UUID], version=4)
+        fsize = file[FileObjectKeys.SIZE]
+        fhash = file[FileObjectKeys.HASH]
+        ftype = file[FileObjectKeys.TYPE].split(".")[-1]
+        fdate = file[FileObjectKeys.DATE]
+        created_at = _parse_datetime(file[FileObjectKeys.CREATED_AT])
+        updated_at = _parse_datetime(file[FileObjectKeys.UPDATED_AT])
+        state = _parse_file_state(file[FileObjectKeys.STATE])
+        categories = file[FileObjectKeys.CATEGORIES]
+
+        mission_id, mission_name = _get_nested_info(file, MISSION)
+        project_id, project_name = _get_nested_info(file[MISSION], PROJECT)
+
+        parsed = File(
+            id=id_,
+            name=name,
+            hash=fhash,
+            size=fsize,
+            type_=ftype,
+            date=fdate,
+            categories=categories,
+            state=state,
+            created_at=created_at,
+            updated_at=updated_at,
+            mission_id=mission_id,
+            mission_name=mission_name,
+            project_id=project_id,
+            project_name=project_name,
+        )
+    except Exception as e:
+        raise ParsingError(f"error parsing file: {file}") from e
+    return parsed

{kleinkram-0.38.1.dev20241212075157 → kleinkram-0.38.1.dev20250207122632}/kleinkram/api/file_transfer.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 import logging
 import sys
-from concurrent.futures import as_completed
 from concurrent.futures import Future
 from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import as_completed
 from enum import Enum
 from pathlib import Path
 from time import monotonic
@@ -17,9 +17,11 @@ from uuid import UUID
 import boto3.s3.transfer
 import botocore.config
 import httpx
+from rich.console import Console
+from tqdm import tqdm
+
 from kleinkram.api.client import AuthenticatedClient
-from kleinkram.config import Config
-from kleinkram.config import LOCAL_S3
+from kleinkram.config import get_config
 from kleinkram.errors import AccessDenied
 from kleinkram.models import File
 from kleinkram.models import FileState
@@ -27,9 +29,6 @@ from kleinkram.utils import b64_md5
 from kleinkram.utils import format_error
 from kleinkram.utils import format_traceback
 from kleinkram.utils import styled_string
-from rich.console import Console
-from tqdm import tqdm
-
 
 logger = logging.getLogger(__name__)
 
@@ -52,16 +51,6 @@ class UploadCredentials(NamedTuple):
     bucket: str
 
 
-def _get_s3_endpoint() -> str:
-    config = Config()
-    endpoint = config.endpoint
-
-    if "localhost" in endpoint:
-        return LOCAL_S3
-    else:
-        return endpoint.replace("api", "minio")
-
-
 def _confirm_file_upload(
     client: AuthenticatedClient, file_id: UUID, file_hash: str
 ) -> None:
@@ -106,7 +95,7 @@ def _get_upload_creditials(
     resp = client.post(UPLOAD_CREDS, json=dct)
     resp.raise_for_status()
 
-    data = resp.json()[0]
+    data = resp.json()["data"][0]
 
     if data.get("error") == FILE_EXISTS_ERROR:
         return None
@@ -170,10 +159,13 @@ def upload_file(
     filename: str,
     path: Path,
     verbose: bool = False,
+    s3_endpoint: Optional[str] = None,
 ) -> UploadState:
     """\
     returns bytes uploaded
     """
+    if s3_endpoint is None:
+        s3_endpoint = get_config().endpoint.s3
 
     total_size = path.stat().st_size
     with tqdm(
@@ -184,8 +176,6 @@ def upload_file(
         leave=False,
         disable=not verbose,
     ) as pbar:
-        endpoint = _get_s3_endpoint()
-
         # get per file upload credentials
         creds = _get_upload_creditials(
             client, internal_filename=filename, mission_id=mission_id
@@ -194,12 +184,11 @@ def upload_file(
             return UploadState.EXISTS
 
         try:
-            _s3_upload(path, endpoint=endpoint, credentials=creds, pbar=pbar)
+            _s3_upload(path, endpoint=s3_endpoint, credentials=creds, pbar=pbar)
         except Exception as e:
             logger.error(format_traceback(e))
             _cancel_file_upload(client, creds.file_id, mission_id)
             return UploadState.CANCELED
-
         else:
             _confirm_file_upload(client, creds.file_id, b64_md5(path))
             return UploadState.UPLOADED
@@ -379,14 +368,14 @@ def _download_handler(
 
 def upload_files(
     client: AuthenticatedClient,
-    files_map: Dict[str, Path],
+    files: Dict[str, Path],
     mission_id: UUID,
     *,
     verbose: bool = False,
     n_workers: int = 2,
 ) -> None:
     with tqdm(
-        total=len(files_map),
+        total=len(files),
         unit="files",
         desc="uploading files",
         disable=not verbose,
@@ -395,7 +384,7 @@ def upload_files(
         start = monotonic()
         futures: Dict[Future[UploadState], Path] = {}
         with ThreadPoolExecutor(max_workers=n_workers) as executor:
-            for name, path in files_map.items():
+            for name, path in files.items():
                 future = executor.submit(
                     upload_file,
                     client=client,

kleinkram-0.38.1.dev20250207122632/kleinkram/api/pagination.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+from typing import Dict
+from typing import Generator
+from typing import List
+from typing import Mapping
+from typing import Optional
+from typing import cast
+
+from kleinkram.api.client import AuthenticatedClient
+
+DataPage = Dict[str, Any]
+
+
+PAGE_SIZE = 128
+SKIP = "skip"
+TAKE = "take"
+
+
+def paginated_request(
+    client: AuthenticatedClient,
+    endpoint: str,
+    params: Optional[Mapping[str, Any]] = None,
+    max_entries: Optional[int] = None,
+    page_size: int = PAGE_SIZE,
+) -> Generator[DataPage, None, None]:
+    total_entries_count = 0
+
+    params = dict(params or {})
+
+    params[TAKE] = page_size
+    params[SKIP] = 0
+
+    while True:
+        resp = client.get(endpoint, params=params)
+        resp.raise_for_status()  # TODO: this is fine for now
+
+        paged_data = resp.json()
+        data_page = cast(List[DataPage], paged_data["data"])
+
+        for entry in data_page:
+            total_entries_count += 1
+            yield entry
+            if max_entries is not None and max_entries <= total_entries_count:
+                return
+
+        count = cast(int, paged_data["count"])
+        skip = cast(int, paged_data["skip"])
+        take = cast(int, paged_data["take"])
+
+        if count - skip - take <= 0:
+            return
+
+        params[SKIP] = total_entries_count

kleinkram-0.38.1.dev20250207122632/kleinkram/api/query.py

@@ -0,0 +1,111 @@
+"""\
+this file contains dataclasses for specifying remote resources on kleinkram
+here we also provide some helper functions to validate certain properties
+of these specifications
+
+additionally we provide wrappers around the api for fetching the specified
+resources (TODO: move this part to another file)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import List
+from uuid import UUID
+
+from kleinkram.errors import InvalidMissionQuery
+from kleinkram.errors import InvalidProjectQuery
+
+MAX_PARALLEL_REQUESTS = 32
+SPECIAL_PATTERN_CHARS = ["*", "?", "[", "]"]
+
+
+@dataclass
+class ProjectQuery:
+    patterns: List[str] = field(default_factory=list)
+    ids: List[UUID] = field(default_factory=list)
+
+
+@dataclass
+class MissionQuery:
+    patterns: List[str] = field(default_factory=list)
+    ids: List[UUID] = field(default_factory=list)
+    project_query: ProjectQuery = field(default=ProjectQuery())
+
+
+@dataclass
+class FileQuery:
+    patterns: List[str] = field(default_factory=list)
+    ids: List[UUID] = field(default_factory=list)
+    mission_query: MissionQuery = field(default=MissionQuery())
+
+
+def check_mission_query_is_creatable(query: MissionQuery) -> str:
+    """\
+    check if a query is unique and can be used to create a mission
+    returns: the mission name
+    """
+    if not mission_query_is_unique(query):
+        raise InvalidMissionQuery(f"Mission query is not unique: {query}")
+    # cant create a missing by id
+    if query.ids:
+        raise InvalidMissionQuery(f"cant create mission by id: {query}")
+    return query.patterns[0]
+
+
+def check_project_query_is_creatable(query: ProjectQuery) -> str:
+    if not project_query_is_unique(query):
+        raise InvalidProjectQuery(f"Project query is not unique: {query}")
+    # cant create a missing by id
+    if query.ids:
+        raise InvalidProjectQuery(f"cant create project by id: {query}")
+    return query.patterns[0]
+
+
+def _pattern_is_unique(pattern: str) -> bool:
+    for char in SPECIAL_PATTERN_CHARS:
+        if char in pattern:
+            return False
+    return True
+
+
+def project_query_is_unique(query: ProjectQuery) -> bool:
+    # a single project id is specified
+    if len(query.ids) == 1 and not query.patterns:
+        return True
+
+    # a single project name is specified
+    if len(query.patterns) == 1 and _pattern_is_unique(query.patterns[0]):
+        return True
+    return False
+
+
+def mission_query_is_unique(query: MissionQuery) -> bool:
+    # a single mission id is specified
+    if len(query.ids) == 1 and not query.patterns:
+        return True
+
+    # a single mission name a unique project spec are specified
+    if (
+        project_query_is_unique(query.project_query)
+        and len(query.patterns) == 1
+        and _pattern_is_unique(query.patterns[0])
+    ):
+        return True
+    return False
+
+
+def file_query_is_unique(query: FileQuery) -> bool:
+    # a single file id is specified
+    if len(query.ids) == 1 and not query.patterns:
+        return True
+
+    # a single file name a unique mission spec are specified
+    if (
+        mission_query_is_unique(query.mission_query)
+        and len(query.patterns) == 1
+        and _pattern_is_unique(query.patterns[0])
+    ):
+        return True
+    return False