kleinkram 0.36.2__py3-none-any.whl → 0.36.2.dev20241118065826__py3-none-any.whl

kleinkram/utils.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+import base64
+import glob
+import hashlib
+import os
+import string
+from hashlib import md5
+from pathlib import Path
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import NamedTuple
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+from typing import Union
+from uuid import UUID
+
+import yaml
+from kleinkram._version import __version__
+from kleinkram.errors import FileTypeNotSupported
+from kleinkram.errors import InvalidFileSpec
+from kleinkram.errors import InvalidMissionSpec
+from kleinkram.models import FilesById
+from kleinkram.models import FilesByMission
+from kleinkram.models import MissionById
+from kleinkram.models import MissionByName
+from rich.console import Console
+
+
+INTERNAL_ALLOWED_CHARS = string.ascii_letters + string.digits + "_" + "-"
+
+
+def check_file_paths(files: Sequence[Path]) -> None:
+    for file in files:
+        if file.is_dir():
+            raise FileNotFoundError(f"{file} is a directory and not a file")
+        if not file.exists():
+            raise FileNotFoundError(f"{file} does not exist")
+        if file.suffix not in (".bag", ".mcap"):
+            raise FileTypeNotSupported(
+                f"only `.bag` or `.mcap` files are supported: {file}"
+            )
+
+
+def raw_rich(*objects: Any, **kwargs: Any) -> str:
+    """\
+    accepts any object that Console.print can print
+    returns the raw string output
+    """
+
+    console = Console()
+
+    with console.capture() as capture:
+        console.print(*objects, **kwargs, end="")
+
+    return capture.get()
+
+
+def is_valid_uuid4(uuid: str) -> bool:
+    try:
+        UUID(uuid, version=4)
+        return True
+    except ValueError:
+        return False
+
+
+def get_filename(path: Path) -> str:
+    """\
+    takes a path and returns a sanitized filename
+
+    the format for this internal filename is:
+    - replace all disallowed characters with "_"
+    - trim to 40 chars + 10 hashed chars
+        - the 10 hashed chars are deterministic given the original filename
+    """
+
+    stem = "".join(
+        char if char in INTERNAL_ALLOWED_CHARS else "_" for char in path.stem
+    )
+
+    if len(stem) > 50:
+        hash = md5(path.name.encode()).hexdigest()
+        stem = f"{stem[:40]}{hash[:10]}"
+
+    return f"{stem}{path.suffix}"
+
+
+def get_filename_map(file_paths: Sequence[Path]) -> Dict[str, Path]:
+    """\
+    takes a list of unique filepaths and returns a mapping
+    from the original filename to a sanitized internal filename
+    see `get_filename` for the internal filename format
+    """
+
+    if len(file_paths) != len(set(file_paths)):
+        raise ValueError("files paths must be unique")
+
+    internal_file_map = {}
+    for file in file_paths:
+        if file.is_dir():
+            raise ValueError(f"got dir {file} expected file")
+
+        internal_file_map[get_filename(file)] = file
+
+    if len(internal_file_map) != len(set(internal_file_map.values())):
+        raise RuntimeError("hash collision")
+
+    return internal_file_map
+
+
+def b64_md5(file: Path) -> str:
+    hash_md5 = hashlib.md5()
+    with open(file, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    binary_digest = hash_md5.digest()
+    return base64.b64encode(binary_digest).decode("utf-8")
+
+
+def get_valid_mission_spec(
+    mission: Union[str, UUID],
+    project: Optional[Union[str, UUID]] = None,
+) -> Union[MissionById, MissionByName]:
+    """\
+    checks if:
+    - atleast one is speicifed
+    - if project is not specified then mission must be a valid uuid4
+    """
+
+    if isinstance(mission, UUID):
+        return MissionById(id=mission)
+    if isinstance(mission, str) and project is not None:
+        return MissionByName(name=mission, project=project)
+    raise InvalidMissionSpec("must specify mission id or project name / id")
+
+
+def get_valid_file_spec(
+    files: Sequence[Union[str, UUID]],
+    mission: Optional[Union[str, UUID]] = None,
+    project: Optional[Union[str, UUID]] = None,
+) -> Union[FilesById, FilesByMission]:
+    """\
+    """
+    if not any([project, mission, files]):
+        raise InvalidFileSpec("must specify `project`, `mission` or `files`")
+
+    # if only files are specified they must be valid uuid4
+    if project is None and mission is None:
+        if all(map(lambda file: isinstance(file, UUID), files)):
+            return FilesById(ids=files)  # type: ignore
+        raise InvalidFileSpec("if no mission is specified files must be valid uuid4")
+
+    if mission is None:
+        raise InvalidMissionSpec("mission must be specified")
+    mission_spec = get_valid_mission_spec(mission, project)
+    return FilesByMission(mission=mission_spec, files=list(files))
+
+
+def to_name_or_uuid(s: str) -> Union[UUID, str]:
+    if is_valid_uuid4(s):
+        return UUID(s)
+    return s
+
+
+def load_metadata(path: Path) -> Dict[str, str]:
+    if not path.exists():
+        raise FileNotFoundError(f"metadata file not found: {path}")
+    try:
+        with path.open() as f:
+            return {str(k): str(v) for k, v in yaml.safe_load(f).items()}
+    except Exception as e:
+        raise ValueError(f"could not parse metadata file: {e}")
+
+
+def get_supported_api_version() -> Tuple[int, int, int]:
+    vers = __version__.split(".")
+    return tuple(map(int, vers[:3]))  # type: ignore

{kleinkram-0.36.2.dist-info/licenses → kleinkram-0.36.2.dev20241118065826.dist-info}/LICENSE

@@ -671,4 +671,4 @@ into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<https://www.gnu.org/licenses/why-not-lgpl.html>.
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

kleinkram-0.36.2.dev20241118065826.dist-info/METADATA

@@ -0,0 +1,113 @@
+Metadata-Version: 2.1
+Name: kleinkram
+Version: 0.36.2.dev20241118065826
+Summary: give me your bags
+Author: Cyrill Püntener, Dominique Garmier, Johann Schwabe
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: boto3
+Requires-Dist: botocore
+Requires-Dist: httpx
+Requires-Dist: pyyaml
+Requires-Dist: rich
+Requires-Dist: tqdm
+Requires-Dist: typer
+
+# Kleinkram: CLI
+
+Install the package
+
+```bash
+pip install kleinkram
+```
+
+Run the CLI
+
+```bash
+klein
+```
+
+## Usage
+
+Here are some basic examples of how to use the CLI.
+
+### Listing Files
+
+To list all files in a mission:
+
+```bash
+klein list -p project-name -m mission-name
+```
+
+### Uploading Files
+
+To upload all `*.bag` files in the current directory to a mission:
+
+```bash
+klein upload -p project-name -m mission-name *.bag
+```
+
+If you would like to create a new mission on upload use the `--create` flag.
+
+### Downloading Files
+
+To download all files from a mission and save them `out`:
+
+```bash
+klein download -p project-name -m mission-name --dest out
+```
+
+You can additionally specify filenames or ids if you only want to download specific files.
+
+Instead of downloading files from a specified mission you can download arbitrary files by specifying their ids:
+
+```bash
+klein download --dest out *id1* *id2* *id3*
+```
+
+For more information consult the [documentation](https://docs.datasets.leggedrobotics.com/usage/cli/cli-getting-started.html).
+
+## Development
+
+Clone the repo
+
+```bash
+git clone git@github.com:leggedrobotics/kleinkram.git
+cd kleinkram/cli
+```
+
+Setup the environment
+
+```bash
+virtualenv -ppython3.8 .venv
+source .venv/bin/activate
+pip install -e . -r requirements.txt
+```
+
+Install `pre-commit` hooks
+
+```bash
+pre-commit install
+```
+
+Run the CLI
+
+```bash
+klein --help
+```
+
+### Run Tests
+```bash
+pytest .
+```
+
+You can also look in `scripts` for some scripts that might be useful for testing.

kleinkram-0.36.2.dev20241118065826.dist-info/RECORD

@@ -0,0 +1,33 @@
+kleinkram/__init__.py,sha256=adI-FqfuuH56n8U-lJa3uZkH_saxi7dr4Hox_HBuf9A,107
+kleinkram/__main__.py,sha256=B9RiZxfO4jpCmWPUHyKJ7_EoZlEG4sPpH-nz7T_YhhQ,125
+kleinkram/_version.py,sha256=QYJyRTcqFcJj4qWYpqs7WcoOP6jxDMqyvxLY-cD6KcE,129
+kleinkram/app.py,sha256=C5NwoFJ4FsUZwx6h5PInjq6j-NveGBgfbpHQ9Vb8Igo,5553
+kleinkram/auth.py,sha256=QAwT3VWdHEv5cxNnRbZoVykYwdeaoZjtBQBBXfejFdQ,2953
+kleinkram/config.py,sha256=QZWBXOCRjXa-wcSynJkCaYSE_0kU85j2zsdp-miR-eU,4447
+kleinkram/consts.py,sha256=70GELDssDm-YeIbChBRTscnNRcyWRUDWWlYBjVWNLDk,205
+kleinkram/core.py,sha256=XmNhg1i520w-9et8r0LSQRsVN09gJgSXfyHRV1tu2rM,227
+kleinkram/enums.py,sha256=bX0H8_xkh_7DRycJYII8MTjHmjbcFW9ZJ3tzlzRhT2k,149
+kleinkram/errors.py,sha256=bZ92LLweQaorsKZWWsL7BAPxFxODQ01nEeenHOHKrv0,1200
+kleinkram/main.py,sha256=hmFuSnE2p-E4nZ2zDVToOsoELZhosLN0r4wsXnbmsas,168
+kleinkram/models.py,sha256=dOrvK-H6cW8tPbZUbBeE6MROp17c5hp5FeULKZRpZ3s,4329
+kleinkram/utils.py,sha256=72PuaARDOlrxTjh5sHS6llvZng2q78j4IpiUP1RG1qA,5335
+kleinkram/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kleinkram/api/client.py,sha256=b1mUwpu7zrXVTNjTDfW8aA0hgmsisRIlMOqQTSbd7bg,2142
+kleinkram/api/file_transfer.py,sha256=qUnJMv7fNgyCvP3czHrytMdW33kvW2FGbAXf7eDtDZc,8934
+kleinkram/api/routes.py,sha256=ojAB5Kfnofw6oyeAe_9VS__-YahpWGsiBWl4jjbQO10,12537
+kleinkram/commands/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
+kleinkram/commands/download.py,sha256=393n9kaJJJmsAJGdPTrkd-H2giNsJuihr1JD1mdF6rk,3415
+kleinkram/commands/endpoint.py,sha256=VgaleKTev8Muu4OmS4tptbhrbl58u1UTAxEmoLBCQew,1642
+kleinkram/commands/list.py,sha256=QEhD6mSDn_dVc9XI_5HpBrfjG93IowzFxH4rOyVHjCI,2666
+kleinkram/commands/mission.py,sha256=BXnGUnOBTSHZipy50P2NtornXUAxDCdyqmGpVUMzfxc,1751
+kleinkram/commands/project.py,sha256=q-kJ1YQr-Axo-Zl0qe2J7eGGgKlSbcdDjpAJIynRLaM,587
+kleinkram/commands/upload.py,sha256=Wc3OkmdtbrloTA4FaE2i1BD2gBcsjkR4T8G9qlIqe-M,4450
+kleinkram/commands/verify.py,sha256=CGndDooBIdnYGPpaaBRNvs-3SWoBUquzAJc-3YB6_Yo,3692
+tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/test_utils.py,sha256=1OOimNDLfLU5IWSwCtJRrXLw4SCbEe5rV4Z2XA7wCiU,4611
+kleinkram-0.36.2.dev20241118065826.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+kleinkram-0.36.2.dev20241118065826.dist-info/METADATA,sha256=xd4wVG3UEjsBcGeOnqTM1jA8bkSyVLXkXINVoe0eusA,2335
+kleinkram-0.36.2.dev20241118065826.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+kleinkram-0.36.2.dev20241118065826.dist-info/entry_points.txt,sha256=SaB2l5aqhSr8gmaMw2kvQU90a8Bnl7PedU8cWYxkfYo,46
+kleinkram-0.36.2.dev20241118065826.dist-info/top_level.txt,sha256=G1Lj9vHAtZn402Ukkrfll-6BCmnDNy_HVtWeNvXzdDA,16
+kleinkram-0.36.2.dev20241118065826.dist-info/RECORD,,

{kleinkram-0.36.2.dist-info → kleinkram-0.36.2.dev20241118065826.dist-info}/WHEEL

@@ -1,4 +1,5 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: setuptools (75.5.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
+

kleinkram-0.36.2.dev20241118065826.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+klein = kleinkram.main:main

kleinkram-0.36.2.dev20241118065826.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+kleinkram
+tests

tests/test_utils.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from uuid import uuid4
+
+import pytest
+from kleinkram.errors import FileTypeNotSupported
+from kleinkram.errors import InvalidFileSpec
+from kleinkram.errors import InvalidMissionSpec
+from kleinkram.models import FilesById
+from kleinkram.models import FilesByMission
+from kleinkram.models import MissionById
+from kleinkram.models import MissionByName
+from kleinkram.utils import b64_md5
+from kleinkram.utils import check_file_paths
+from kleinkram.utils import get_filename
+from kleinkram.utils import get_filename_map
+from kleinkram.utils import get_valid_file_spec
+from kleinkram.utils import get_valid_mission_spec
+from kleinkram.utils import is_valid_uuid4
+from kleinkram.utils import to_name_or_uuid
+
+
+def test_check_file_paths():
+    with TemporaryDirectory() as temp_dir:
+        exits_txt = Path(temp_dir) / "exists.txt"
+        exists_bag = Path(temp_dir) / "exists.bag"
+        exits_mcap = Path(temp_dir) / "exists.mcap"
+        not_exists = Path(temp_dir) / "not_exists.txt"
+        is_dir = Path(temp_dir) / "is_dir"
+
+        exits_txt.touch()
+        exists_bag.touch()
+        exits_mcap.touch()
+        is_dir.mkdir()
+
+        with pytest.raises(FileTypeNotSupported):
+            check_file_paths([exits_txt])
+
+        with pytest.raises(FileNotFoundError):
+            check_file_paths([not_exists])
+
+        with pytest.raises(FileNotFoundError):
+            check_file_paths([is_dir])
+
+        assert check_file_paths([exists_bag, exits_mcap]) is None
+
+
+def test_is_valid_uuid4():
+    valid = "e896313b-2ab0-466b-b458-8911575fdee9"
+    invalid = "hello world"
+
+    assert is_valid_uuid4(valid)
+    assert not is_valid_uuid4(invalid)
+
+
+@pytest.mark.parametrize(
+    "old, new",
+    [
+        pytest.param(Path("test.bar"), "test.bar", id="short name"),
+        pytest.param(Path("symbols_-123.txt"), "symbols_-123.txt", id="symbols"),
+        pytest.param(
+            Path("invalid sybmols $%^&.txt"),
+            "invalid_sybmols_____.txt",
+            id="invalid symbols",
+        ),
+        pytest.param(
+            Path(f'{"a" * 100}.txt'), f'{"a" * 40}38bf3e475f.txt', id="too long"
+        ),
+        pytest.param(Path(f'{"a" * 50}.txt'), f'{"a" * 50}.txt', id="max length"),
+        pytest.param(Path("in/a/folder.txt"), "folder.txt", id="in folder"),
+    ],
+)
+def test_get_filename(old, new):
+    assert get_filename(old) == new
+
+
+def test_get_filename_map():
+    non_unique = [Path("a.txt"), Path("a.txt")]
+
+    with pytest.raises(ValueError):
+        get_filename_map(non_unique)
+
+    unique = [Path("a.txt"), Path("b.txt")]
+    assert get_filename_map(unique) == {get_filename(Path(p)): Path(p) for p in unique}
+
+
+def test_b64_md5():
+    with TemporaryDirectory() as temp_dir:
+        file = Path(temp_dir) / "file.txt"
+        file.write_text("hello world")
+
+        assert b64_md5(file) == "XrY7u+Ae7tCTyyK7j1rNww=="
+
+
+def test_get_valid_mission_spec():
+    # only mission name
+    with pytest.raises(InvalidMissionSpec):
+        get_valid_mission_spec("mission")
+
+    # only mission id
+    id_ = uuid4()
+    assert get_valid_mission_spec(id_) == MissionById(id_)
+
+    # mission name and project name
+    assert get_valid_mission_spec("mission", "project") == MissionByName(
+        "mission", "project"
+    )
+
+    # mission name and project id
+    project_id = uuid4()
+    assert get_valid_mission_spec("mission", project_id) == MissionByName(
+        "mission", project_id
+    )
+
+    # mission id and project name
+    assert get_valid_mission_spec(id_, "project") == MissionById(id_)
+
+
+def test_get_valid_file_spec():
+    # no information
+    with pytest.raises(InvalidFileSpec):
+        get_valid_file_spec([], None, None)
+
+    # only file ids
+    file_ids = [uuid4(), uuid4()]
+    assert get_valid_file_spec(file_ids, None, None) == FilesById(file_ids)
+
+    # only file names
+    with pytest.raises(InvalidFileSpec):
+        get_valid_file_spec(["foo"], None, None)
+
+    # missing mission
+    with pytest.raises(InvalidMissionSpec):
+        get_valid_file_spec([], None, "project")
+
+    # mission name and file names
+    assert get_valid_file_spec([], "mission", "project") == FilesByMission(
+        MissionByName("mission", "project"), []
+    )
+
+    assert get_valid_file_spec(
+        file_ids + ["name"], "mission", "project"
+    ) == FilesByMission(MissionByName("mission", "project"), file_ids + ["name"])
+
+
+def test_to_name_or_uuid():
+    id_ = uuid4()
+    not_id = "not an id"
+
+    assert to_name_or_uuid(str(id_)) == id_
+    assert to_name_or_uuid(not_id) == not_id

kleinkram/api_client.py

@@ -1,63 +0,0 @@
-import httpx
-
-from kleinkram.auth.auth import TokenFile, CLI_KEY, AUTH_TOKEN, REFRESH_TOKEN
-
-
-class NotAuthenticatedException(Exception):
-
-    def __init__(self, endpoint: str):
-        self.message = f"You are not authenticated on endpoint '{endpoint}'. Please run 'klein login' to authenticate."
-
-
-class AuthenticatedClient(httpx.Client):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        try:
-            self.tokenfile = TokenFile()
-            self._load_cookies()
-        except Exception:
-            raise NotAuthenticatedException(self.tokenfile.endpoint)
-
-    def _load_cookies(self):
-        if self.tokenfile.isCliToken():
-            self.cookies.set(CLI_KEY, self.tokenfile.getCLIToken())
-        else:
-            self.cookies.set(AUTH_TOKEN, self.tokenfile.getAuthToken())
-
-    def refresh_token(self):
-        if self.tokenfile.isCliToken():
-            print("CLI key cannot be refreshed.")
-            return
-        refresh_token = self.tokenfile.getRefreshToken()
-        if not refresh_token:
-            print("No refresh token found. Please login again.")
-            raise Exception("No refresh token found.")
-        self.cookies.set(
-            REFRESH_TOKEN,
-            refresh_token,
-        )
-        response = self.post(
-            "/auth/refresh-token",
-        )
-        response.raise_for_status()
-        new_access_token = response.cookies.get(AUTH_TOKEN)
-        new_tokens = {AUTH_TOKEN: new_access_token, REFRESH_TOKEN: refresh_token}
-        self.tokenfile.saveTokens(new_tokens)
-        self.cookies.set(AUTH_TOKEN, new_access_token)
-
-    def request(self, method, url, *args, **kwargs):
-        response = super().request(
-            method, self.tokenfile.endpoint + url, *args, **kwargs
-        )
-        if (url == "/auth/refresh-token") and response.status_code == 401:
-            print("Refresh token expired. Please login again.")
-            response.status_code = 403
-            exit(1)
-        if response.status_code == 401:
-            print("Token expired, refreshing token...")
-            self.refresh_token()
-            response = super().request(
-                method, self.tokenfile.endpoint + url, *args, **kwargs
-            )
-        return response