hirundo 0.1.7__py3-none-any.whl → 0.1.8__py3-none-any.whl

hirundo/__init__.py

@@ -32,4 +32,4 @@ __all__ = [
     "StorageIntegration",
 ]
 
-__version__ = "0.1.7"
+__version__ = "0.1.8"

hirundo/_env.py

@@ -1,8 +1,19 @@
+import enum
 import os
+from pathlib import Path
 
 from dotenv import load_dotenv
 
-load_dotenv()
+
+class EnvLocation(enum.Enum):
+    DOTENV = Path.cwd() / ".env"
+    HOME = Path.home() / ".hirundo.conf"
+
+
+if os.path.exists(EnvLocation.DOTENV.value):
+    load_dotenv(EnvLocation.DOTENV.value)
+elif os.path.exists(EnvLocation.HOME.value):
+    load_dotenv(EnvLocation.HOME.value)
 
 API_HOST = os.getenv("API_HOST", "https://api.hirundo.io")
 API_KEY = os.getenv("API_KEY")

hirundo/_http.py

@@ -0,0 +1,14 @@
+from requests import Response
+
+import hirundo.logger
+
+logger = hirundo.logger.get_logger(__name__)
+
+
+def raise_for_status_with_reason(response: Response):
+    try:
+        response.reason = response.json().get("reason", None)
+    except Exception as e:
+        logger.debug("Failed to parse response as JSON: %s", e)
+
+    response.raise_for_status()

hirundo/_iter_sse_retrying.py

@@ -1,7 +1,7 @@
 import asyncio
 import time
+import typing
 from collections.abc import AsyncGenerator, Generator
-from typing import Union
 
 import httpx
 from httpx_sse import ServerSentEvent, aconnect_sse, connect_sse
@@ -13,7 +13,7 @@ def iter_sse_retrying(
     client: httpx.Client,
     method: str,
     url: str,
-    headers: Union[dict[str, str], None] = None,
+    headers: typing.Optional[dict[str, str]] = None,
 ) -> Generator[ServerSentEvent, None, None]:
     if headers is None:
         headers = {}

hirundo/cli.py

@@ -1,11 +1,14 @@
+import os
 import re
 import sys
+import typing
+from pathlib import Path
 from typing import Annotated
 from urllib.parse import urlparse
 
 import typer
 
-from hirundo._env import API_HOST
+from hirundo._env import API_HOST, EnvLocation
 
 docs = "sphinx" in sys.modules
 hirundo_epilog = (
@@ -23,7 +26,9 @@ app = typer.Typer(
 )
 
 
-def upsert_env(var_name: str, var_value: str):
+def _upsert_env(
+    dotenv_filepath: typing.Union[str, Path], var_name: str, var_value: str
+):
     """
     Change an environment variable in the .env file.
     If the variable does not exist, it will be added.
@@ -32,18 +37,30 @@ def upsert_env(var_name: str, var_value: str):
         var_name: The name of the environment variable to change.
         var_value: The new value of the environment variable.
     """
-    dotenv = "./.env"
     regex = re.compile(rf"^{var_name}=.*$")
-    with open(dotenv) as f:
-        lines = f.readlines()
+    lines = []
+    if os.path.exists(dotenv_filepath):
+        with open(dotenv_filepath) as f:
+            lines = f.readlines()
 
-    with open(dotenv, "w") as f:
+    with open(dotenv_filepath, "w") as f:
         f.writelines(line for line in lines if not regex.search(line) and line != "\n")
 
-    with open(dotenv, "a") as f:
+    with open(dotenv_filepath, "a") as f:
         f.writelines(f"\n{var_name}={var_value}")
 
 
+def upsert_env(var_name: str, var_value: str):
+    if os.path.exists(EnvLocation.DOTENV.value):
+        # If a `.env` file exists, re-use it
+        _upsert_env(EnvLocation.DOTENV.value, var_name, var_value)
+        return EnvLocation.DOTENV.name
+    else:
+        # Create a `.hirundo.conf` file with environment variables in the home directory
+        _upsert_env(EnvLocation.HOME.value, var_name, var_value)
+        return EnvLocation.HOME.name
+
+
 def fix_api_host(api_host: str):
     if not api_host.startswith("http") and not api_host.startswith("https"):
         api_host = f"https://{api_host}"
@@ -72,8 +89,15 @@ def setup_api_key(
     Setup the API key for the Hirundo client library.
     Values are saved to a .env file in the current directory for use by the library in requests.
     """
-    upsert_env("API_KEY", api_key)
-    print("API key saved to .env for future use. Please do not share the .env file")
+    saved_to = upsert_env("API_KEY", api_key)
+    if saved_to == EnvLocation.HOME.name:
+        print(
+            "API key saved to ~/.hirundo.conf for future use. Please do not share the ~/.hirundo.conf file since it contains your secret API key."
+        )
+    elif saved_to == EnvLocation.DOTENV.name:
+        print(
+            "API key saved to local .env file for future use. Please do not share the .env file since it contains your secret API key."
+        )
 
 
 @app.command("change-remote", epilog=hirundo_epilog)
@@ -94,8 +118,13 @@ def change_api_remote(
     """
     api_host = fix_api_host(api_host)
 
-    upsert_env("API_HOST", api_host)
-    print("API host saved to .env for future use. Please do not share this file")
+    saved_to = upsert_env("API_HOST", api_host)
+    if saved_to == EnvLocation.HOME.name:
+        print(
+            "API host saved to ~/.hirundo.conf for future use. Please do not share the ~/.hirundo.conf file"
+        )
+    elif saved_to == EnvLocation.DOTENV.name:
+        print("API host saved to .env for future use. Please do not share this file")
 
 
 @app.command("setup", epilog=hirundo_epilog)
@@ -123,11 +152,41 @@ def setup(
     Setup the Hirundo client library.
     """
     api_host = fix_api_host(api_host)
-    upsert_env("API_HOST", api_host)
-    upsert_env("API_KEY", api_key)
-    print(
-        "API host and API key saved to .env for future use. Please do not share this file"
-    )
+    api_host_saved_to = upsert_env("API_HOST", api_host)
+    api_key_saved_to = upsert_env("API_KEY", api_key)
+    if api_host_saved_to != api_key_saved_to:
+        print(
+            "API host and API key saved to different locations. This should not happen. Please report this issue."
+        )
+        if (
+            api_host_saved_to == EnvLocation.HOME.name
+            and api_key_saved_to == EnvLocation.DOTENV.name
+        ):
+            print(
+                "API host saved to ~/.hirundo.conf for future use. Please do not share the ~/.hirundo.conf file"
+            )
+            print(
+                "API key saved to local .env file for future use. Please do not share the .env file since it contains your secret API key."
+            )
+        elif (
+            api_host_saved_to == EnvLocation.DOTENV.name
+            and api_key_saved_to == EnvLocation.HOME.name
+        ):
+            print(
+                "API host saved to .env for future use. Please do not share this file"
+            )
+            print(
+                "API key saved to ~/.hirundo.conf for future use. Please do not share the ~/.hirundo.conf file since it contains your secret API key."
+            )
+        return
+    if api_host_saved_to == EnvLocation.HOME.name:
+        print(
+            "API host and API key saved to ~/.hirundo.conf for future use. Please do not share the ~/.hirundo.conf file since it contains your secret API key."
+        )
+    elif api_host_saved_to == EnvLocation.DOTENV.name:
+        print(
+            "API host and API key saved to .env for future use. Please do not share this file since it contains your secret API key."
+        )
 
 
 typer_click_object = typer.main.get_command(app)

hirundo/dataset_optimization.py

@@ -3,17 +3,20 @@ import typing
 from collections.abc import AsyncGenerator, Generator
 from enum import Enum
 from io import StringIO
-from typing import Union, overload
+from typing import overload
 
 import httpx
+import numpy as np
 import pandas as pd
 import requests
+from pandas._typing import DtypeArg
 from pydantic import BaseModel, Field, model_validator
 from tqdm import tqdm
 from tqdm.contrib.logging import logging_redirect_tqdm
 
 from hirundo._env import API_HOST
 from hirundo._headers import get_auth_headers, json_headers
+from hirundo._http import raise_for_status_with_reason
 from hirundo._iter_sse_retrying import aiter_sse_retrying, iter_sse_retrying
 from hirundo._timeouts import MODIFY_TIMEOUT, READ_TIMEOUT
 from hirundo.enum import DatasetMetadataType, LabellingType
@@ -40,6 +43,58 @@ class RunStatus(Enum):
     SUCCESS = "SUCCESS"
     FAILURE = "FAILURE"
     AWAITING_MANUAL_APPROVAL = "AWAITING MANUAL APPROVAL"
+    RETRYING = "RETRYING"
+
+
+STATUS_TO_TEXT_MAP = {
+    RunStatus.STARTED.value: "Optimization run in progress. Downloading dataset",
+    RunStatus.PENDING.value: "Optimization run queued and not yet started",
+    RunStatus.SUCCESS.value: "Optimization run completed successfully",
+    RunStatus.FAILURE.value: "Optimization run failed",
+    RunStatus.AWAITING_MANUAL_APPROVAL.value: "Awaiting manual approval",
+    RunStatus.RETRYING.value: "Optimization run failed. Retrying",
+}
+STATUS_TO_PROGRESS_MAP = {
+    RunStatus.STARTED.value: 0.0,
+    RunStatus.PENDING.value: 0.0,
+    RunStatus.SUCCESS.value: 100.0,
+    RunStatus.FAILURE.value: 100.0,
+    RunStatus.AWAITING_MANUAL_APPROVAL.value: 100.0,
+    RunStatus.RETRYING.value: 0.0,
+}
+
+
+class DatasetOptimizationResults(BaseModel):
+    model_config = {"arbitrary_types_allowed": True}
+
+    suspects: pd.DataFrame
+    """
+    A pandas DataFrame containing the results of the optimization run
+    """
+    warnings_and_errors: pd.DataFrame
+    """
+    A pandas DataFrame containing the warnings and errors of the optimization run
+    """
+
+
+CUSTOMER_INTERCHANGE_DTYPES: DtypeArg = {
+    "image_path": str,
+    "label_path": str,
+    "segments_mask_path": str,
+    "segment_id": np.int32,
+    "label": str,
+    "bbox_id": str,
+    "xmin": np.int32,
+    "ymin": np.int32,
+    "xmax": np.int32,
+    "ymax": np.int32,
+    "suspect_level": np.float32,  # If exists, must be one of the values in the enum below
+    "suggested_label": str,
+    "suggested_label_conf": np.float32,
+    "status": str,
+    # ⬆️ If exists, must be one of the following:
+    # NO_LABELS/MISSING_IMAGE/INVALID_IMAGE/INVALID_BBOX/INVALID_BBOX_SIZE/INVALID_SEG/INVALID_SEG_SIZE
+}
 
 
 class OptimizationDataset(BaseModel):
@@ -54,13 +109,13 @@ class OptimizationDataset(BaseModel):
     - `LabellingType.SingleLabelClassification`: Indicates that the dataset is for classification tasks
     - `LabellingType.ObjectDetection`: Indicates that the dataset is for object detection tasks
     """
-    dataset_storage: Union[StorageLink, None]
+    dataset_storage: typing.Optional[StorageLink]
     """
     The storage link to the dataset. This can be a link to a file or a directory containing the dataset.
     If `None`, the `dataset_id` field must be set.
     """
 
-    classes: list[str]
+    classes: typing.Optional[list[str]] = None
     """
     A full list of possible classes used in classification / object detection.
     It is currently required for clarity and performance.
@@ -78,15 +133,15 @@ class OptimizationDataset(BaseModel):
     Currently no other formats are supported. Future versions of `hirundo` may support additional formats.
     """
 
-    storage_integration_id: Union[int, None] = Field(default=None, init=False)
+    storage_integration_id: typing.Optional[int] = Field(default=None, init=False)
     """
     The ID of the storage integration used to store the dataset and metadata.
     """
-    dataset_id: Union[int, None] = Field(default=None, init=False)
+    dataset_id: typing.Optional[int] = Field(default=None, init=False)
     """
     The ID of the dataset created on the server.
     """
-    run_id: Union[str, None] = Field(default=None, init=False)
+    run_id: typing.Optional[str] = Field(default=None, init=False)
     """
     The ID of the Dataset Optimization run created on the server.
     """
@@ -98,7 +153,7 @@ class OptimizationDataset(BaseModel):
         return self
 
     @staticmethod
-    def list(organization_id: Union[int, None] = None) -> list[dict]:
+    def list(organization_id: typing.Optional[int] = None) -> list[dict]:
         """
         Lists all the `OptimizationDataset` instances created by user's default organization
         or the `organization_id` passed
@@ -113,7 +168,7 @@ class OptimizationDataset(BaseModel):
             headers=get_auth_headers(),
             timeout=READ_TIMEOUT,
         )
-        response.raise_for_status()
+        raise_for_status_with_reason(response)
         return response.json()
 
     @staticmethod
@@ -129,7 +184,7 @@ class OptimizationDataset(BaseModel):
             headers=get_auth_headers(),
             timeout=MODIFY_TIMEOUT,
         )
-        response.raise_for_status()
+        raise_for_status_with_reason(response)
         logger.info("Deleted dataset with ID: %s", dataset_id)
 
     def delete(self, storage_integration=True) -> None:
@@ -184,7 +239,7 @@ class OptimizationDataset(BaseModel):
             },
             timeout=MODIFY_TIMEOUT,
         )
-        dataset_response.raise_for_status()
+        raise_for_status_with_reason(dataset_response)
         self.dataset_id = dataset_response.json()["id"]
         if not self.dataset_id:
             raise HirundoError("Failed to create the dataset")
@@ -208,7 +263,7 @@ class OptimizationDataset(BaseModel):
             headers=get_auth_headers(),
             timeout=MODIFY_TIMEOUT,
         )
-        run_response.raise_for_status()
+        raise_for_status_with_reason(run_response)
         return run_response.json()["run_id"]
 
     def run_optimization(self) -> str:
@@ -274,10 +329,19 @@ class OptimizationDataset(BaseModel):
         return df
 
     @staticmethod
-    def _read_csv_to_df(data: dict):
+    def _read_csvs_to_df(data: dict):
         if data["state"] == RunStatus.SUCCESS.value:
-            data["result"] = OptimizationDataset._clean_df_index(
-                pd.read_csv(StringIO(data["result"]))
+            data["result"]["suspects"] = OptimizationDataset._clean_df_index(
+                pd.read_csv(
+                    StringIO(data["result"]["suspects"]),
+                    dtype=CUSTOMER_INTERCHANGE_DTYPES,
+                )
+            )
+            data["result"]["warnings_and_errors"] = OptimizationDataset._clean_df_index(
+                pd.read_csv(
+                    StringIO(data["result"]["warnings_and_errors"]),
+                    dtype=CUSTOMER_INTERCHANGE_DTYPES,
+                )
             )
         else:
             pass
@@ -307,7 +371,7 @@ class OptimizationDataset(BaseModel):
                 if not last_event:
                     continue
                 data = last_event["data"]
-                OptimizationDataset._read_csv_to_df(data)
+                OptimizationDataset._read_csvs_to_df(data)
                 yield data
         if not last_event or last_event["data"]["state"] == RunStatus.PENDING.value:
             OptimizationDataset._check_run_by_id(run_id, retry + 1)
@@ -316,27 +380,24 @@ class OptimizationDataset(BaseModel):
     @overload
     def check_run_by_id(
         run_id: str, stop_on_manual_approval: typing.Literal[True]
-    ) -> typing.Optional[pd.DataFrame]:
-        ...
+    ) -> typing.Optional[DatasetOptimizationResults]: ...
 
     @staticmethod
     @overload
     def check_run_by_id(
         run_id: str, stop_on_manual_approval: typing.Literal[False] = False
-    ) -> pd.DataFrame:
-        ...
+    ) -> DatasetOptimizationResults: ...
 
     @staticmethod
     @overload
     def check_run_by_id(
         run_id: str, stop_on_manual_approval: bool
-    ) -> typing.Optional[pd.DataFrame]:
-        ...
+    ) -> typing.Optional[DatasetOptimizationResults]: ...
 
     @staticmethod
     def check_run_by_id(
         run_id: str, stop_on_manual_approval: bool = False
-    ) -> typing.Optional[pd.DataFrame]:
+    ) -> typing.Optional[DatasetOptimizationResults]:
         """
         Check the status of a run given its ID
 
@@ -345,7 +406,7 @@ class OptimizationDataset(BaseModel):
             stop_on_manual_approval: If True, the function will return `None` if the run is awaiting manual approval
 
         Returns:
-            A pandas DataFrame with the results of the optimization run
+            A DatasetOptimizationResults object with the results of the optimization run
 
         Raises:
             HirundoError: If the maximum number of retries is reached or if the run fails
@@ -354,22 +415,29 @@ class OptimizationDataset(BaseModel):
         with logging_redirect_tqdm():
             t = tqdm(total=100.0)
             for iteration in OptimizationDataset._check_run_by_id(run_id):
-                if iteration["state"] == RunStatus.SUCCESS.value:
-                    t.set_description("Optimization run completed successfully")
-                    t.n = 100.0
-                    t.refresh()
-                    t.close()
-                    return iteration["result"]
-                elif iteration["state"] == RunStatus.PENDING.value:
-                    t.set_description("Optimization run queued and not yet started")
-                    t.n = 0.0
-                    t.refresh()
-                elif iteration["state"] == RunStatus.STARTED.value:
-                    t.set_description(
-                        "Optimization run in progress. Downloading dataset"
-                    )
-                    t.n = 0.0
+                if iteration["state"] in STATUS_TO_PROGRESS_MAP:
+                    t.set_description(STATUS_TO_TEXT_MAP[iteration["state"]])
+                    t.n = STATUS_TO_PROGRESS_MAP[iteration["state"]]
+                    logger.debug("Setting progress to %s", t.n)
                     t.refresh()
+                    if iteration["state"] == RunStatus.FAILURE.value:
+                        raise HirundoError(
+                            f"Optimization run failed with error: {iteration['result']}"
+                        )
+                    elif iteration["state"] == RunStatus.SUCCESS.value:
+                        t.close()
+                        return DatasetOptimizationResults(
+                            suspects=iteration["result"]["suspects"],
+                            warnings_and_errors=iteration["result"][
+                                "warnings_and_errors"
+                            ],
+                        )
+                    elif (
+                        iteration["state"] == RunStatus.AWAITING_MANUAL_APPROVAL.value
+                        and stop_on_manual_approval
+                    ):
+                        t.close()
+                        return None
                 elif iteration["state"] is None:
                     if (
                         iteration["result"]
@@ -387,37 +455,23 @@ class OptimizationDataset(BaseModel):
                         )
                         t.set_description(desc)
                         t.n = current_progress_percentage
+                        logger.debug("Setting progress to %s", t.n)
                         t.refresh()
-                elif iteration["state"] == RunStatus.AWAITING_MANUAL_APPROVAL.value:
-                    t.set_description("Awaiting manual approval")
-                    t.n = 100.0
-                    t.refresh()
-                    if stop_on_manual_approval:
-                        t.close()
-                        return None
-                elif iteration["state"] == RunStatus.FAILURE.value:
-                    t.set_description("Optimization run failed")
-                    t.close()
-                    raise HirundoError(
-                        f"Optimization run failed with error: {iteration['result']}"
-                    )
         raise HirundoError("Optimization run failed with an unknown error")
 
     @overload
     def check_run(
         self, stop_on_manual_approval: typing.Literal[True]
-    ) -> typing.Union[pd.DataFrame, None]:
-        ...
+    ) -> typing.Optional[DatasetOptimizationResults]: ...
 
     @overload
     def check_run(
         self, stop_on_manual_approval: typing.Literal[False] = False
-    ) -> pd.DataFrame:
-        ...
+    ) -> DatasetOptimizationResults: ...
 
     def check_run(
         self, stop_on_manual_approval: bool = False
-    ) -> typing.Union[pd.DataFrame, None]:
+    ) -> typing.Optional[DatasetOptimizationResults]:
         """
         Check the status of the current active instance's run.
 
@@ -511,7 +565,7 @@ class OptimizationDataset(BaseModel):
             headers=get_auth_headers(),
             timeout=MODIFY_TIMEOUT,
         )
-        response.raise_for_status()
+        raise_for_status_with_reason(response)
 
     def cancel(self) -> None:
         """

hirundo/git.py

@@ -1,5 +1,6 @@
 import re
-from typing import Annotated, Union
+import typing
+from typing import Annotated
 
 import pydantic
 import requests
@@ -8,6 +9,7 @@ from pydantic_core import Url
 
 from hirundo._env import API_HOST
 from hirundo._headers import get_auth_headers, json_headers
+from hirundo._http import raise_for_status_with_reason
 from hirundo._timeouts import MODIFY_TIMEOUT, READ_TIMEOUT
 from hirundo.logger import get_logger
 
@@ -30,14 +32,14 @@ class GitSSHAuthBase(BaseModel):
     """
     The SSH key for the Git repository
     """
-    ssh_password: Union[str, None]
+    ssh_password: typing.Optional[str]
     """
     The password for the SSH key for the Git repository.
     """
 
 
 class GitRepo(BaseModel):
-    id: Union[int, None] = None
+    id: typing.Optional[int] = None
     """
     The ID of the Git repository.
     """
@@ -51,20 +53,20 @@ class GitRepo(BaseModel):
     The URL of the Git repository, it should start with `ssh://` or `https://` or be in the form `user@host:path`.
     If it is in the form `user@host:path`, it will be rewritten to `ssh://user@host:path`.
     """
-    organization_id: Union[int, None] = None
+    organization_id: typing.Optional[int] = None
     """
     The ID of the organization that the Git repository belongs to.
     If not provided, it will be assigned to your default organization.
     """
 
-    plain_auth: Union[GitPlainAuthBase, None] = pydantic.Field(
+    plain_auth: typing.Optional[GitPlainAuthBase] = pydantic.Field(
         default=None, examples=[None, {"username": "ben", "password": "password"}]
     )
     """
     The plain authentication details for the Git repository.
     Use this if using a special user with a username and password for authentication.
     """
-    ssh_auth: Union[GitSSHAuthBase, None] = pydantic.Field(
+    ssh_auth: typing.Optional[GitSSHAuthBase] = pydantic.Field(
         default=None,
         examples=[
             {
@@ -112,7 +114,7 @@ class GitRepo(BaseModel):
             },
             timeout=MODIFY_TIMEOUT,
         )
-        git_repo.raise_for_status()
+        raise_for_status_with_reason(git_repo)
         git_repo_id = git_repo.json()["id"]
         self.id = git_repo_id
         return git_repo_id
@@ -129,7 +131,7 @@ class GitRepo(BaseModel):
             },
             timeout=READ_TIMEOUT,
         )
-        git_repos.raise_for_status()
+        raise_for_status_with_reason(git_repos)
         return git_repos.json()
 
     @staticmethod
@@ -147,7 +149,7 @@ class GitRepo(BaseModel):
             },
             timeout=MODIFY_TIMEOUT,
         )
-        git_repo.raise_for_status()
+        raise_for_status_with_reason(git_repo)
 
     def delete(self):
         """

hirundo/logger.py

@@ -1,8 +1,10 @@
 import logging
+import os
 
 
 def get_logger(name: str) -> logging.Logger:
     logger = logging.getLogger(name)
-    logger.setLevel(logging.INFO)
+    log_level = os.getenv("LOG_LEVEL")
+    logger.setLevel(log_level if log_level else logging.INFO)
     logger.addHandler(logging.StreamHandler())
     return logger

hirundo/storage.py

@@ -1,6 +1,5 @@
 import typing
 from enum import Enum
-from typing import Union
 
 import pydantic
 import requests
@@ -10,6 +9,7 @@ from pydantic_core import Url
 from hirundo._constraints import S3BucketUrl, StorageIntegrationName
 from hirundo._env import API_HOST
 from hirundo._headers import get_auth_headers, json_headers
+from hirundo._http import raise_for_status_with_reason
 from hirundo._timeouts import MODIFY_TIMEOUT, READ_TIMEOUT
 from hirundo.git import GitRepo
 from hirundo.logger import get_logger
@@ -18,18 +18,18 @@ logger = get_logger(__name__)
 
 
 class StorageS3(BaseModel):
-    endpoint_url: Union[Url, None] = None
+    endpoint_url: typing.Optional[Url] = None
     bucket_url: S3BucketUrl
     region_name: str
     # ⬆️ We could restrict this, but if we're allowing custom endpoints then the validation may be wrong
-    access_key_id: Union[str, None] = None
-    secret_access_key: Union[str, None] = None
+    access_key_id: typing.Optional[str] = None
+    secret_access_key: typing.Optional[str] = None
 
 
 class StorageGCP(BaseModel):
     bucket_name: str
     project: str
-    credentials_json: Union[dict, None] = None
+    credentials_json: typing.Optional[dict] = None
 
 
 # TODO: Azure storage integration is coming soon
@@ -40,12 +40,12 @@ class StorageGCP(BaseModel):
 
 
 class StorageGit(BaseModel):
-    repo_id: Union[int, None] = None
+    repo_id: typing.Optional[int] = None
     """
     The ID of the Git repository in the Hirundo system.
     Either `repo_id` or `repo` must be provided.
     """
-    repo: Union[GitRepo, None] = None
+    repo: typing.Optional[GitRepo] = None
     """
     The Git repository to link to.
     Either `repo_id` or `repo` must be provided.
@@ -79,9 +79,9 @@ class StorageTypes(str, Enum):
 
 
 class StorageIntegration(BaseModel):
-    id: Union[int, None] = None
+    id: typing.Optional[int] = None
 
-    organization_id: Union[int, None] = None
+    organization_id: typing.Optional[int] = None
     """
     The ID of the organization that the `StorageIntegration` belongs to.
     If not provided, it will be assigned to your default organization.
@@ -107,7 +107,7 @@ class StorageIntegration(BaseModel):
     - `Azure` (coming soon)
     - `Git`
     """
-    s3: Union[StorageS3, None] = pydantic.Field(
+    s3: typing.Optional[StorageS3] = pydantic.Field(
         default=None,
         examples=[
             {
@@ -125,7 +125,7 @@ class StorageIntegration(BaseModel):
     The Amazon Web Services (AWS) S3 storage integration details.
     Use this if you want to link to an S3 bucket.
     """
-    gcp: Union[StorageGCP, None] = pydantic.Field(
+    gcp: typing.Optional[StorageGCP] = pydantic.Field(
         default=None,
         examples=[
             None,
@@ -155,7 +155,7 @@ class StorageIntegration(BaseModel):
     Use this if you want to link to an GCS bucket.
     """
     azure: None = None
-    # azure: Union[StorageAzure, None] = pydantic.Field(
+    # azure: typing.Optional[StorageAzure] = pydantic.Field(
     #     default=None,
     #     examples=[
     #         None,
@@ -168,7 +168,7 @@ class StorageIntegration(BaseModel):
     #         None,
     #     ],
     # )  TODO: Azure storage integration is coming soon
-    git: Union[StorageGit, None] = pydantic.Field(
+    git: typing.Optional[StorageGit] = pydantic.Field(
         default=None,
         examples=[
             None,
@@ -191,7 +191,7 @@ class StorageIntegration(BaseModel):
     """
 
     @staticmethod
-    def list(organization_id: typing.Union[int, None] = None) -> list[dict]:
+    def list(organization_id: typing.Optional[int] = None) -> list[dict]:
         """
         Lists all the `StorageIntegration`'s created by user's default organization
         Note: The return type is `list[dict]` and not `list[StorageIntegration]`
@@ -206,7 +206,7 @@ class StorageIntegration(BaseModel):
             headers=get_auth_headers(),
             timeout=READ_TIMEOUT,
         )
-        storage_integrations.raise_for_status()
+        raise_for_status_with_reason(storage_integrations)
         return storage_integrations.json()
 
     @staticmethod
@@ -222,7 +222,7 @@ class StorageIntegration(BaseModel):
             headers=get_auth_headers(),
             timeout=MODIFY_TIMEOUT,
         )
-        storage_integration.raise_for_status()
+        raise_for_status_with_reason(storage_integration)
         logger.info("Deleted storage integration with ID: %s", storage_integration_id)
 
     def delete(self) -> None:
@@ -248,7 +248,7 @@ class StorageIntegration(BaseModel):
             },
             timeout=MODIFY_TIMEOUT,
         )
-        storage_integration.raise_for_status()
+        raise_for_status_with_reason(storage_integration)
         storage_integration_id = storage_integration.json()["id"]
         self.id = storage_integration_id
         logger.info("Created storage integration with ID: %s", storage_integration_id)

hirundo-0.1.8.dist-info/METADATA

@@ -0,0 +1,176 @@
+Metadata-Version: 2.1
+Name: hirundo
+Version: 0.1.8
+Summary: This package is used to interface with Hirundo's platform. It provides a simple API to optimize your ML datasets.
+Author-email: Hirundo <dev@hirundo.io>
+License: MIT License
+        
+        Copyright (c) 2024, Hirundo
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/Hirundo-io/hirundo-client
+Keywords: dataset,machine learning,data science,data engineering
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: types-PyYAML>=6.0.12
+Requires-Dist: pydantic>=2.7.1
+Requires-Dist: twine>=5.0.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: types-requests>=2.31.0
+Requires-Dist: typer>=0.12.3
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: stamina>=24.2.0
+Requires-Dist: httpx-sse>=0.4.0
+Requires-Dist: pandas>=2.2.2
+Requires-Dist: tqdm>=4.66.5
+Provides-Extra: dev
+Requires-Dist: pyyaml>=6.0.1; extra == "dev"
+Requires-Dist: types-PyYAML>=6.0.12; extra == "dev"
+Requires-Dist: pydantic>=2.7.1; extra == "dev"
+Requires-Dist: twine>=5.0.0; extra == "dev"
+Requires-Dist: python-dotenv>=1.0.1; extra == "dev"
+Requires-Dist: types-requests>=2.31.0; extra == "dev"
+Requires-Dist: types-setuptools>=69.5.0; extra == "dev"
+Requires-Dist: typer>=0.12.3; extra == "dev"
+Requires-Dist: httpx>=0.27.0; extra == "dev"
+Requires-Dist: stamina>=24.2.0; extra == "dev"
+Requires-Dist: httpx-sse>=0.4.0; extra == "dev"
+Requires-Dist: pytest>=8.2.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.6; extra == "dev"
+Requires-Dist: uv; extra == "dev"
+Requires-Dist: pre-commit>=3.7.1; extra == "dev"
+Requires-Dist: ruff==0.6.5; extra == "dev"
+Requires-Dist: bumpver; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7.4.7; extra == "docs"
+Requires-Dist: sphinx-autobuild>=2024.4.16; extra == "docs"
+Requires-Dist: sphinx-click>=5.0.1; extra == "docs"
+Requires-Dist: autodoc-pydantic>=2.2.0; extra == "docs"
+Requires-Dist: furo; extra == "docs"
+Requires-Dist: sphinx-multiversion; extra == "docs"
+
+# Hirundo
+
+This package exposes access to Hirundo APIs for dataset optimization for Machine Learning.
+
+Dataset optimization is currently available for datasets labelled for classification and object detection.
+
+
+Support dataset storage integrations include:
+   - Google Cloud (GCP) Storage
+   - Amazon Web Services (AWS) S3
+   - Git LFS (Large File Storage) repositories (e.g. GitHub or HuggingFace)
+
+Optimizing a classification dataset
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Currently ``hirundo`` requires a CSV file with the following columns (all columns are required):
+   - ``image_path``: The location of the image within the dataset ``root``
+   - ``label``: The label of the image, i.e. which the class that was annotated for this image
+
+And outputs a CSV with the same columns and:
+   - ``suspect_level``: mislabel suspect level
+   - ``suggested_label``: suggested label
+   - ``suggested_label_conf``: suggested label confidence
+
+Optimizing an object detection (OD) dataset
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Currently ``hirundo`` requires a CSV file with the following columns (all columns are required):
+   - ``image_path``: The location of the image within the dataset ``root``
+   - ``bbox_id``: The index of the bounding box within the dataset. Used to indicate label suspects
+   - ``label``: The label of the image, i.e. which the class that was annotated for this image
+   - ``x1``, ``y1``, ``x2``, ``y2``: The bounding box coordinates of the object within the image
+
+And outputs a CSV with the same columns and:
+   - ``suspect_level``: object mislabel suspect level
+   - ``suggested_label``: suggested object label
+   - ``suggested_label_conf``: suggested object label confidence
+
+Note: This Python package must be used alongside a Hirundo server, either the SaaS platform, a custom VPC deployment or an on-premises installation.
+
+
+## Installation
+
+You can install the codebase with a simple `pip install hirundo` to install the latest version of this package. If you prefer to install from the Git repository and/or need a specific version or branch, you can simply clone the repository, check out the relevant commit and then run `pip install .` to install that version. A full list of dependencies can be found in `requirements.txt`, but these will be installed automatically by either of these commands.
+
+## Usage
+
+Classification example:
+```
+from hirundo.dataset_optimization import OptimizationDataset
+from hirundo.enum import LabellingType
+from hirundo.storage import StorageIntegration, StorageLink, StorageTypes
+
+test_dataset = OptimizationDataset(
+    name="TEST-GCP cifar 100 classification dataset",
+    labelling_type=LabellingType.SingleLabelClassification,
+    dataset_storage=StorageLink(
+        storage_integration=StorageIntegration(
+            name="cifar100bucket",
+            type=StorageTypes.GCP,
+            gcp=StorageGCP(
+                bucket_name="cifar100bucket",
+                project="Hirundo-global",
+                credentials_json=json.loads(os.environ["GCP_CREDENTIALS"]),
+            ),
+        ),
+        path="/pytorch-cifar/data",
+    ),
+    dataset_metadata_path="cifar100.csv",
+    classes=cifar100_classes,
+)
+
+test_dataset.run_optimization()
+results = test_dataset.check_run()
+print(results)
+```
+
+
+Object detection example:
+
+```
+from hirundo.dataset_optimization import OptimizationDataset
+from hirundo.enum import LabellingType
+from hirundo.storage import StorageIntegration, StorageLink, StorageTypes
+
+test_dataset = OptimizationDataset(
+    name=f"TEST-HuggingFace-BDD-100k-validation-OD-validation-dataset{unique_id}",
+    labelling_type=LabellingType.ObjectDetection,
+    dataset_storage=StorageLink(
+        storage_integration=StorageIntegration(
+            name=f"BDD-100k-validation-dataset{unique_id}",
+            type=StorageTypes.GIT,
+            git=StorageGit(
+                repo=GitRepo(
+                    name=f"BDD-100k-validation-dataset{unique_id}",
+                    repository_url="https://git@hf.co/datasets/hirundo-io/bdd100k-validation-only",
+                ),
+                branch="main",
+            ),
+        ),
+        path="/BDD100K Val from Hirundo.zip/bdd100k",
+    ),
+    dataset_metadata_path="bdd100k.csv",
+)
+
+test_dataset.run_optimization()
+results = test_dataset.check_run()
+print(results)
+```
+
+Note: Currently we only support the main CPython release 3.9, 3.10 and 3.11. PyPy support may be introduced in the future.
+
+## Further documentation
+
+To learn about mroe how to use this library, please visit the [http://docs.hirundo.io/](documentation) or see the Google Colab examples.

hirundo-0.1.8.dist-info/RECORD

@@ -0,0 +1,20 @@
+hirundo/__init__.py,sha256=EINZmdlmNjdW_dM85wksapRxGL-pPC49OYvKUBRtxQk,707
+hirundo/__main__.py,sha256=wcCrL4PjG51r5wVKqJhcoJPTLfHW0wNbD31DrUN0MWI,28
+hirundo/_constraints.py,sha256=-RAUV9GnCsaT9pLGSqYglKOeK0joPBBexGTo87j5nkI,425
+hirundo/_env.py,sha256=dXUFPeEL1zPe-eBdWD4_WZvlgiY2cpWuVDzf41Qjuto,609
+hirundo/_headers.py,sha256=ggTyBwVT3nGyPidCcmYMX6pv0idzMxCI2S1BJQE-Bbs,253
+hirundo/_http.py,sha256=INrHX7ncpXS9vdyjrske3B5vUKL5ke9SIY6daffahtE,350
+hirundo/_iter_sse_retrying.py,sha256=0u-jJe5vHCZegImKBB1rpI9O1BnN7oWJytdabl34ih4,3345
+hirundo/_timeouts.py,sha256=IfX8-mrLp809-A_xSLv1DhIqZnO-Qvy4FcTtOtvqLog,42
+hirundo/cli.py,sha256=4-pdV483zqRJl8d-R9p_9YOGlehOnoMJzb3XAAdPRb0,6634
+hirundo/dataset_optimization.py,sha256=I2AzkSns_MLwlwI4mGGxaJB6OUG3pv7VJ5uFAtcJdTM,21825
+hirundo/enum.py,sha256=-3w09g-_yRYIMiM8VA_Nb07WoQXf5IjyERTGonzNDs0,457
+hirundo/git.py,sha256=Dbp0ALJYhLDgkmI_5u9iVyE_xEHIxoUTeZdpU8iau_4,4884
+hirundo/logger.py,sha256=MUqrYp0fBlxWFhGl6P5t19_uqO7T_PNhrLN5bqY3i7s,275
+hirundo/storage.py,sha256=xifT6xuFCJpVp5wB-ZZkzKz9HbVcMNrllj10vXlU1vU,9845
+hirundo-0.1.8.dist-info/LICENSE,sha256=fusGGjqT2RGlU6kbkaOk7d-gDnsjk17wq67AO0mwBZI,1065
+hirundo-0.1.8.dist-info/METADATA,sha256=heoP6t876hsxEih-RzaIjGtcLZl8UOpcwExnjQ8thU4,7841
+hirundo-0.1.8.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+hirundo-0.1.8.dist-info/entry_points.txt,sha256=4ZtnA_Nl1Af8fLnHp3lwjbGDEGU1S6ujb_JwtuQ7ZPM,44
+hirundo-0.1.8.dist-info/top_level.txt,sha256=cmyNqrNZOAYxnywJGFI1AJBLe4SkH8HGsfFx6ncdrbI,8
+hirundo-0.1.8.dist-info/RECORD,,

{hirundo-0.1.7.dist-info → hirundo-0.1.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.2)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

hirundo-0.1.7.dist-info/METADATA

@@ -1,118 +0,0 @@
-Metadata-Version: 2.1
-Name: hirundo
-Version: 0.1.7
-Summary: This package is used to interface with Hirundo's platform. It provides a simple API to optimize your ML datasets.
-Author-email: Hirundo <dev@hirundo.io>
-License: MIT License
-        
-        Copyright (c) 2024, Hirundo
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-        
-Project-URL: Homepage, https://github.com/Hirundo-io/hirundo-client
-Keywords: dataset,machine learning,data science,data engineering
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: pyyaml>=6.0.1
-Requires-Dist: types-PyYAML>=6.0.12
-Requires-Dist: pydantic>=2.7.1
-Requires-Dist: twine>=5.0.0
-Requires-Dist: python-dotenv>=1.0.1
-Requires-Dist: types-requests>=2.31.0
-Requires-Dist: typer>=0.12.3
-Requires-Dist: httpx>=0.27.0
-Requires-Dist: stamina>=24.2.0
-Requires-Dist: httpx-sse>=0.4.0
-Requires-Dist: pandas>=2.2.2
-Requires-Dist: tqdm>=4.66.5
-Provides-Extra: dev
-Requires-Dist: pyyaml>=6.0.1; extra == "dev"
-Requires-Dist: types-PyYAML>=6.0.12; extra == "dev"
-Requires-Dist: pydantic>=2.7.1; extra == "dev"
-Requires-Dist: twine>=5.0.0; extra == "dev"
-Requires-Dist: python-dotenv>=1.0.1; extra == "dev"
-Requires-Dist: types-requests>=2.31.0; extra == "dev"
-Requires-Dist: types-setuptools>=69.5.0; extra == "dev"
-Requires-Dist: typer>=0.12.3; extra == "dev"
-Requires-Dist: httpx>=0.27.0; extra == "dev"
-Requires-Dist: stamina>=24.2.0; extra == "dev"
-Requires-Dist: httpx-sse>=0.4.0; extra == "dev"
-Requires-Dist: pytest>=8.2.0; extra == "dev"
-Requires-Dist: pytest-asyncio>=0.23.6; extra == "dev"
-Requires-Dist: uv; extra == "dev"
-Requires-Dist: pre-commit>=3.7.1; extra == "dev"
-Requires-Dist: ruff; extra == "dev"
-Requires-Dist: bumpver; extra == "dev"
-Provides-Extra: docs
-Requires-Dist: sphinx>=7.4.7; extra == "docs"
-Requires-Dist: sphinx-autobuild>=2024.4.16; extra == "docs"
-Requires-Dist: sphinx-click>=5.0.1; extra == "docs"
-Requires-Dist: autodoc-pydantic>=2.2.0; extra == "docs"
-Requires-Dist: furo; extra == "docs"
-Requires-Dist: sphinx-multiversion; extra == "docs"
-
-# Hirundo client
-
-This repo contains the source code for the Hirundo client library
-
-## Usage:
-
-To learn about how to use this library, please visit the [http://docs.hirundo.io/](documentation) or see the Google Colab examples.
-
-Note: Currently we only support the main CPython release 3.9, 3.10 and 3.11. PyPy support may be introduced in the future.
-
-## Development:
-
-### Install dev dependencies
-
-```bash
-pip install -r dev-requirements.txt
-```
-
-Note: You can install and use `uv` as a faster drop-in replacement for `pip`. We have it as part of our dev dependencies for this reason.
-
-### Install `git` hooks (optional)
-
-```bash
-pre-commit install
-```
-
-### Check lint and apply formatting with Ruff (optional; pre-commit hooks run this automatically)
-
-```bash
-ruff check
-ruff format
-```
-
-### Change packages
-
-#### Update `requirements.txt` files
-
-```bash
-uv pip compile pyproject.toml
-uv pip compile --extra dev -o dev-requirements.txt -c requirements.txt pyproject.toml
-uv pip compile --extra docs -o docs-requirements.txt -c requirements.txt pyproject.toml
-```
-
-#### Sync installed packages
-
-```bash
-uv pip sync dev-requirements.txt
-```
-
-### Build process
-
-To build the package, run:
-`python -m build`
-
-### Publish documentation & releases
-
-Documentation & releases are published via GitHub Actions on merges to `main`.

hirundo-0.1.7.dist-info/RECORD

@@ -1,19 +0,0 @@
-hirundo/__init__.py,sha256=K0TcpPS937MazldFEgHnWLIKKmf-nt2k6Vjef58aKNs,707
-hirundo/__main__.py,sha256=wcCrL4PjG51r5wVKqJhcoJPTLfHW0wNbD31DrUN0MWI,28
-hirundo/_constraints.py,sha256=-RAUV9GnCsaT9pLGSqYglKOeK0joPBBexGTo87j5nkI,425
-hirundo/_env.py,sha256=6RVEn80KelYa-v2Tc18KTQlAZx0cYv3Y1HFS3H7gDt0,307
-hirundo/_headers.py,sha256=ggTyBwVT3nGyPidCcmYMX6pv0idzMxCI2S1BJQE-Bbs,253
-hirundo/_iter_sse_retrying.py,sha256=WLp_lw8ycBuAxoJkkGBu4y74Ajhcu11r1X-vd5_571A,3352
-hirundo/_timeouts.py,sha256=IfX8-mrLp809-A_xSLv1DhIqZnO-Qvy4FcTtOtvqLog,42
-hirundo/cli.py,sha256=pNRaeH__3qvdfcAWhqnGyZDE4uqtqW8BQ6fy4mD7aM4,3936
-hirundo/dataset_optimization.py,sha256=qevme8Gavuk6dBvR-Q5KffMzt9--hTfxzoUnSx4tqFI,19594
-hirundo/enum.py,sha256=-3w09g-_yRYIMiM8VA_Nb07WoQXf5IjyERTGonzNDs0,457
-hirundo/git.py,sha256=-Z_uFHsFfVR-_XqEq85wiiGZtgqIV81PWoC1B6UBSww,4769
-hirundo/logger.py,sha256=e_Kn6dic8DCqjQnDw60z25xnfofOypNoUdlnunySARs,198
-hirundo/storage.py,sha256=Dfi-LEjfrZJZM9mWE_CgQ4upP2lrN6-3HHGd-0znfqE,9737
-hirundo-0.1.7.dist-info/LICENSE,sha256=fusGGjqT2RGlU6kbkaOk7d-gDnsjk17wq67AO0mwBZI,1065
-hirundo-0.1.7.dist-info/METADATA,sha256=EKswSiESzpSqV9qL9o5uqJNsCUgK7UduvAr_mlXO9hg,4558
-hirundo-0.1.7.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-hirundo-0.1.7.dist-info/entry_points.txt,sha256=4ZtnA_Nl1Af8fLnHp3lwjbGDEGU1S6ujb_JwtuQ7ZPM,44
-hirundo-0.1.7.dist-info/top_level.txt,sha256=cmyNqrNZOAYxnywJGFI1AJBLe4SkH8HGsfFx6ncdrbI,8
-hirundo-0.1.7.dist-info/RECORD,,