hirundo 0.1.3__py3-none-any.whl

hirundo/__init__.py

@@ -0,0 +1,35 @@
+from .dataset_optimization import (
+    HirundoError,
+    OptimizationDataset,
+)
+from .enum import (
+    DatasetMetadataType,
+    LabellingType,
+)
+from .git import GitRepo
+from .storage import (
+    StorageGCP,
+    # StorageAzure,  TODO: Azure storage integration is coming soon
+    StorageGit,
+    StorageIntegration,
+    StorageLink,
+    StorageS3,
+    StorageTypes,
+)
+
+__all__ = [
+    "HirundoError",
+    "OptimizationDataset",
+    "LabellingType",
+    "DatasetMetadataType",
+    "GitRepo",
+    "StorageLink",
+    "StorageTypes",
+    "StorageS3",
+    "StorageGCP",
+    # "StorageAzure",  TODO: Azure storage integration is coming soon
+    "StorageGit",
+    "StorageIntegration",
+]
+
+__version__ = "0.1.3"

hirundo/__main__.py

@@ -0,0 +1,3 @@
+from .cli import app
+
+app()

hirundo/_constraints.py

@@ -0,0 +1,21 @@
+from typing import Annotated
+
+from pydantic import StringConstraints
+
+S3BucketUrl = Annotated[
+    str,
+    StringConstraints(
+        min_length=8,
+        max_length=1023,
+        pattern=r"s3?://[a-z0-9.-]{3,64}[/]?",  # Only allow real S3 bucket URLs
+    ),
+]
+
+StorageIntegrationName = Annotated[
+    str,
+    StringConstraints(
+        min_length=1,
+        max_length=255,
+        pattern=r"^[a-zA-Z0-9-_]+$",
+    ),
+]

hirundo/_env.py

@@ -0,0 +1,12 @@
+import os
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+API_HOST = os.getenv("API_HOST", "https://api.hirundo.io")
+API_KEY = os.getenv("API_KEY")
+if not API_KEY:
+    raise ValueError(
+        "API_KEY is not set. Please run `hirundo setup` to set the API key"
+    )

hirundo/_headers.py

@@ -0,0 +1,9 @@
+from hirundo._env import API_KEY
+
+json_headers = {
+    "Content-Type": "application/json",
+    "Accept": "application/json",
+}
+auth_headers = {
+    "Authorization": f"Bearer {API_KEY}",
+}

hirundo/_iter_sse_retrying.py

@@ -0,0 +1,97 @@
+import asyncio
+import time
+from collections.abc import AsyncGenerator, Generator
+from typing import Union
+
+import httpx
+from httpx_sse import ServerSentEvent, aconnect_sse, connect_sse
+from stamina import retry
+
+
+# Credit: https://github.com/florimondmanca/httpx-sse/blob/master/README.md#handling-reconnections
+def iter_sse_retrying(
+    client: httpx.Client,
+    method: str,
+    url: str,
+    headers: Union[dict[str, str], None] = None,
+) -> Generator[ServerSentEvent, None, None]:
+    if headers is None:
+        headers = {}
+    last_event_id = ""
+    reconnection_delay = 0.0
+
+    # `stamina` will apply jitter and exponential backoff on top of
+    # the `retry` reconnection delay sent by the server.
+    # httpx.ReadError is thrown when there is a network error.
+    #   Some network errors may be temporary, hence the retries.
+    # httpx.RemoteProtocolError is thrown when the server closes the connection.
+    #  This may happen when the server is overloaded and closes the connection or
+    #  when Kubernetes restarts / replaces a pod.
+    #  Likewise, this will likely be temporary, hence the retries.
+    @retry(on=(httpx.ReadError, httpx.RemoteProtocolError))
+    def _iter_sse():
+        nonlocal last_event_id, reconnection_delay
+
+        time.sleep(reconnection_delay)
+
+        connect_headers = {
+            **headers,
+            "Accept": "text/event-stream",
+            "X-Accel-Buffering": "no",
+        }
+
+        if last_event_id:
+            connect_headers["Last-Event-ID"] = last_event_id
+
+        with connect_sse(client, method, url, headers=connect_headers) as event_source:
+            for sse in event_source.iter_sse():
+                last_event_id = sse.id
+
+                if sse.retry is not None:
+                    reconnection_delay = sse.retry / 1000
+
+                yield sse
+
+    return _iter_sse()
+
+
+async def aiter_sse_retrying(
+    client: httpx.AsyncClient,
+    method: str,
+    url: str,
+    headers: dict[str, str],
+) -> AsyncGenerator[ServerSentEvent, None]:
+    last_event_id = ""
+    reconnection_delay = 0.0
+
+    # `stamina` will apply jitter and exponential backoff on top of
+    # the `retry` reconnection delay sent by the server.
+    # httpx.ReadError is thrown when there is a network error.
+    #   Some network errors may be temporary, hence the retries.
+    # httpx.RemoteProtocolError is thrown when the server closes the connection.
+    #  This may happen when the server is overloaded and closes the connection or
+    #  when Kubernetes restarts / replaces a pod.
+    #  Likewise, this will likely be temporary, hence the retries.
+    @retry(on=(httpx.ReadError, httpx.RemoteProtocolError))
+    async def _iter_sse() -> AsyncGenerator[ServerSentEvent, None]:
+        nonlocal last_event_id, reconnection_delay
+
+        await asyncio.sleep(reconnection_delay)
+
+        connect_headers = {**headers, "Accept": "text/event-stream"}
+
+        if last_event_id:
+            connect_headers["Last-Event-ID"] = last_event_id
+
+        async with aconnect_sse(
+            client, method, url, headers=connect_headers
+        ) as event_source:
+            async for sse in event_source.aiter_sse():
+                last_event_id = sse.id
+
+                if sse.retry is not None:
+                    reconnection_delay = sse.retry / 1000
+
+                yield sse
+
+    return _iter_sse()

hirundo/_timeouts.py

@@ -0,0 +1,2 @@
+READ_TIMEOUT = 30.0
+MODIFY_TIMEOUT = 60.0

hirundo/cli.py

@@ -0,0 +1,132 @@
+import re
+import sys
+from typing import Annotated
+from urllib.parse import urlparse
+
+import typer
+
+from hirundo._env import API_HOST
+
+docs = "sphinx" in sys.modules
+hirundo_epilog = (
+    None
+    if docs
+    else "Made with ❤️ by Hirundo. Visit https://www.hirundo.io for more information."
+)
+
+app = typer.Typer(
+    name="hirundo", no_args_is_help=True, rich_markup_mode="rich", epilog=hirundo_epilog
+)
+
+
+def upsert_env(var_name: str, var_value: str):
+    """
+    Change an environment variable in the .env file.
+    If the variable does not exist, it will be added.
+
+    Args:
+        var_name: The name of the environment variable to change.
+        var_value: The new value of the environment variable.
+    """
+    dotenv = "./.env"
+    regex = re.compile(rf"^{var_name}=.*$")
+    with open(dotenv) as f:
+        lines = f.readlines()
+
+    with open(dotenv, "w") as f:
+        f.writelines(line for line in lines if not regex.search(line) and line != "\n")
+
+    with open(dotenv, "a") as f:
+        f.writelines(f"\n{var_name}={var_value}")
+
+
+def fix_api_host(api_host: str):
+    if not api_host.startswith("http") and not api_host.startswith("https"):
+        api_host = f"https://{api_host}"
+        print(
+            "API host must start with 'http://' or 'https://'. Automatically added 'https://'."
+        )
+    if (url := urlparse(api_host)) and url.path != "":
+        print("API host should not contain a path. Removing path.")
+        api_host = f"{url.scheme}://{url.hostname}"
+    return api_host
+
+
+@app.command("set-api-key", epilog=hirundo_epilog)
+def setup_api_key(
+    api_key: Annotated[
+        str,
+        typer.Option(
+            prompt="Please enter the API key value",
+            help=""
+            if docs
+            else f"Visit '{API_HOST}/api-key' to generate your API key.",
+        ),
+    ],
+):
+    """
+    Setup the API key for the Hirundo client library.
+    Values are saved to a .env file in the current directory for use by the library in requests.
+    """
+    upsert_env("API_KEY", api_key)
+    print("API key saved to .env for future use. Please do not share the .env file")
+
+
+@app.command("change-remote", epilog=hirundo_epilog)
+def change_api_remote(
+    api_host: Annotated[
+        str,  # TODO: Change to HttpUrl when https://github.com/tiangolo/typer/pull/723 is merged
+        typer.Option(
+            prompt="Please enter the API server address",
+            help=""
+            if docs
+            else f"Current API server address: '{API_HOST}'. This is the same address where you access the Hirundo web interface.",
+        ),
+    ],
+):
+    """
+    Change the API server address for the Hirundo client library.
+    This is the same address where you access the Hirundo web interface.
+    """
+    api_host = fix_api_host(api_host)
+
+    upsert_env("API_HOST", api_host)
+    print("API host saved to .env for future use. Please do not share this file")
+
+
+@app.command("setup", epilog=hirundo_epilog)
+def setup(
+    api_key: Annotated[
+        str,
+        typer.Option(
+            prompt="Please enter the API key value",
+            help=""
+            if docs
+            else f"Visit '{API_HOST}/api-key' to generate your API key.",
+        ),
+    ],
+    api_host: Annotated[
+        str,  # TODO: Change to HttpUrl as above
+        typer.Option(
+            prompt="Please enter the API server address",
+            help=""
+            if docs
+            else f"Current API server address: '{API_HOST}'. This is the same address where you access the Hirundo web interface.",
+        ),
+    ],
+):
+    """
+    Setup the Hirundo client library.
+    """
+    api_host = fix_api_host(api_host)
+    upsert_env("API_HOST", api_host)
+    upsert_env("API_KEY", api_key)
+    print(
+        "API host and API key saved to .env for future use. Please do not share this file"
+    )
+
+
+typer_click_object = typer.main.get_command(app)
+
+if __name__ == "__main__":
+    app()

hirundo/dataset_optimization.py

@@ -0,0 +1,383 @@
+import json
+import logging
+from collections.abc import AsyncGenerator, Generator
+from typing import Union
+
+import httpx
+import requests
+from pydantic import BaseModel, Field, model_validator
+
+from hirundo._env import API_HOST
+from hirundo._headers import auth_headers, json_headers
+from hirundo._iter_sse_retrying import aiter_sse_retrying, iter_sse_retrying
+from hirundo._timeouts import MODIFY_TIMEOUT, READ_TIMEOUT
+from hirundo.enum import DatasetMetadataType, LabellingType
+from hirundo.storage import StorageIntegration, StorageLink
+
+logger = logging.getLogger(__name__)
+
+
+class HirundoError(Exception):
+    """
+    Custom exception used to indicate errors in `hirundo` dataset optimization runs
+    """
+
+    pass
+
+
+MAX_RETRIES = 200  # Max 200 retries for HTTP SSE connection
+
+
+class OptimizationDataset(BaseModel):
+    name: str
+    """
+    The name of the dataset. Used to identify it amongst the list of datasets
+    belonging to your organization in `hirundo`.
+    """
+    labelling_type: LabellingType
+    """
+    Indicates the labelling type of the dataset. The labelling type can be one of the following:
+    - `LabellingType.SingleLabelClassification`: Indicates that the dataset is for classification tasks
+    - `LabellingType.ObjectDetection`: Indicates that the dataset is for object detection tasks
+    """
+    dataset_storage: Union[StorageLink, None]
+    """
+    The storage link to the dataset. This can be a link to a file or a directory containing the dataset.
+    If `None`, the `dataset_id` field must be set.
+    """
+
+    classes: list[str]
+    """
+    A full list of possible classes used in classification / object detection.
+    It is currently required for clarity and performance.
+    """
+    dataset_metadata_path: str = "metadata.csv"
+    """
+    The path to the dataset metadata file within storage integration, e.g. S3 Bucket / GCP Bucket / Azure Blob storage / Git repo.
+    Note: This path will be prefixed with the `StorageLink`'s `path`.
+    """
+    dataset_metadata_type: DatasetMetadataType = DatasetMetadataType.HirundoCSV
+    """
+    The type of dataset metadata file. The dataset metadata file can be one of the following:
+    - `DatasetMetadataType.HirundoCSV`: Indicates that the dataset metadata file is a CSV file with the Hirundo format
+
+    Currently no other formats are supported. Future versions of `hirundo` may support additional formats.
+    """
+
+    storage_integration_id: Union[int, None] = Field(default=None, init=False)
+    """
+    The ID of the storage integration used to store the dataset and metadata.
+    """
+    dataset_id: Union[int, None] = Field(default=None, init=False)
+    """
+    The ID of the dataset created on the server.
+    """
+    run_id: Union[str, None] = Field(default=None, init=False)
+    """
+    The ID of the Dataset Optimization run created on the server.
+    """
+
+    @model_validator(mode="after")
+    def validate_dataset(self):
+        if self.dataset_storage is None and self.storage_integration_id is None:
+            raise ValueError("No dataset storage has been provided")
+        return self
+
+    @staticmethod
+    def list(organization_id: Union[int, None] = None) -> list[dict]:
+        """
+        Lists all the `OptimizationDataset` instances created by user's default organization
+        or the `organization_id` passed
+        Note: The return type is `list[dict]` and not `list[OptimizationDataset]`
+
+        Args:
+            organization_id: The ID of the organization to list the datasets for.
+        """
+        response = requests.get(
+            f"{API_HOST}/dataset-optimization/dataset/",
+            params={"dataset_organization_id": organization_id},
+            headers=auth_headers,
+            timeout=READ_TIMEOUT,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    @staticmethod
+    def delete_by_id(dataset_id: int) -> None:
+        """
+        Deletes a `OptimizationDataset` instance from the server by its ID
+
+        Args:
+            dataset_id: The ID of the `OptimizationDataset` instance to delete
+        """
+        response = requests.delete(
+            f"{API_HOST}/dataset-optimization/dataset/{dataset_id}",
+            headers=auth_headers,
+            timeout=MODIFY_TIMEOUT,
+        )
+        response.raise_for_status()
+
+    def delete(self, storage_integration=True) -> None:
+        """
+        Deletes the active `OptimizationDataset` instance from the server.
+        It can only be used on a `OptimizationDataset` instance that has been created.
+
+        Args:
+            storage_integration: If True, the `OptimizationDataset`'s `StorageIntegration` will also be deleted
+
+        Note: If `storage_integration` is not set to `False` then the `storage_integration_id` must be set
+        This can either be set manually or by creating the `StorageIntegration` instance via the `OptimizationDataset`'s
+        `create` method
+        """
+        if storage_integration:
+            if not self.storage_integration_id:
+                raise ValueError("No storage integration has been created")
+            StorageIntegration.delete_by_id(self.storage_integration_id)
+        if not self.dataset_id:
+            raise ValueError("No dataset has been created")
+        self.delete_by_id(self.dataset_id)
+
+    def create(self) -> int:
+        """
+        Create a `OptimizationDataset` instance on the server.
+        If `storage_integration_id` is not set, it will be created.
+        """
+        if not self.dataset_storage:
+            raise ValueError("No dataset storage has been provided")
+        if (
+            self.dataset_storage
+            and self.dataset_storage.storage_integration
+            and not self.storage_integration_id
+        ):
+            self.storage_integration_id = (
+                self.dataset_storage.storage_integration.create()
+            )
+        model_dict = self.model_dump()
+        # ⬆️ Get dict of model fields from Pydantic model instance
+        dataset_response = requests.post(
+            f"{API_HOST}/dataset-optimization/dataset/",
+            json={
+                "dataset_storage": {
+                    "storage_integration_id": self.storage_integration_id,
+                    "path": self.dataset_storage.path,
+                },
+                **{k: model_dict[k] for k in model_dict.keys() - {"dataset_storage"}},
+            },
+            headers={
+                **json_headers,
+                **auth_headers,
+            },
+            timeout=MODIFY_TIMEOUT,
+        )
+        dataset_response.raise_for_status()
+        self.dataset_id = dataset_response.json()["id"]
+        if not self.dataset_id:
+            raise HirundoError("Failed to create the dataset")
+        return self.dataset_id
+
+    @staticmethod
+    def launch_optimization_run(dataset_id: int) -> str:
+        """
+        Run the dataset optimization process on the server using the dataset with the given ID
+        i.e. `dataset_id`.
+
+        Args:
+            dataset_id: The ID of the dataset to run optimization on.
+
+        Returns:
+            ID of the run (`run_id`).
+        """
+        run_response = requests.post(
+            f"{API_HOST}/dataset-optimization/run/{dataset_id}",
+            headers=auth_headers,
+            timeout=MODIFY_TIMEOUT,
+        )
+        run_response.raise_for_status()
+        return run_response.json()["run_id"]
+
+    def run_optimization(self) -> str:
+        """
+        If the dataset was not created on the server yet, it is created.
+        Run the dataset optimization process on the server using the active `OptimizationDataset` instance
+
+        Returns:
+            An ID of the run (`run_id`) and stores that `run_id` on the instance
+        """
+        try:
+            if not self.dataset_id:
+                self.dataset_id = self.create()
+            run_id = self.launch_optimization_run(self.dataset_id)
+            self.run_id = run_id
+            return run_id
+        except requests.HTTPError as error:
+            try:
+                content = error.response.json()
+                logger.error(
+                    "HTTP Error! Status code:",
+                    error.response.status_code,
+                    "Content:",
+                    content,
+                )
+            except Exception:
+                content = error.response.text
+            raise HirundoError(
+                f"Failed to start the run. Status code: {error.response.status_code} Content: {content}"
+            ) from error
+        except Exception as error:
+            raise HirundoError(f"Failed to start the run: {error}") from error
+
+    def clean_ids(self):
+        """
+        Reset `dataset_id`, `storage_integration_id`, and `run_id` values on the instance to default value of `None`
+        """
+        self.storage_integration_id = None
+        self.dataset_id = None
+        self.run_id = None
+
+    @staticmethod
+    def check_run_by_id(run_id: str, retry=0) -> Generator[dict, None, None]:
+        """
+        Check the status of a run given its ID
+
+        This generator will produce values to show progress of the run.
+
+        Args:
+            run_id: The `run_id` produced by a `run_optimization` call
+            retry: A number used to track the number of retries to limit re-checks. *Do not* provide this value manually.
+
+        Yields:
+            Each event will be a dict, where:
+            - `"state"` is PENDING, STARTED, RETRY, FAILURE or SUCCESS
+            - `"result"` is a string describing the progress as a percentage for a PENDING state,
+              or the error for a FAILURE state or the results for a SUCCESS state
+
+        """
+        if retry > MAX_RETRIES:
+            raise HirundoError("Max retries reached")
+        last_event = None
+        with httpx.Client(timeout=httpx.Timeout(None, connect=5.0)) as client:
+            for sse in iter_sse_retrying(
+                client,
+                "GET",
+                f"{API_HOST}/dataset-optimization/run/{run_id}",
+                headers=auth_headers,
+            ):
+                if sse.event == "ping":
+                    continue
+                logger.debug(
+                    "[SYNC] received event: %s with data: %s and ID: %s and retry: %s",
+                    sse.event,
+                    sse.data,
+                    sse.id,
+                    sse.retry,
+                )
+                last_event = json.loads(sse.data)
+                yield last_event["data"]
+        if not last_event or last_event["data"]["state"] == "PENDING":
+            OptimizationDataset.check_run_by_id(run_id, retry + 1)
+
+    def check_run(self) -> Generator[dict, None, None]:
+        """
+        Check the status of the current active instance's run.
+
+        This generator will produce values to show progress of the run.
+
+        Yields:
+            Each event will be a dict, where:
+            - `"state"` is PENDING, STARTED, RETRY, FAILURE or SUCCESS
+            - `"result"` is a string describing the progress as a percentage for a PENDING state, or the error for a FAILURE state or the results for a SUCCESS state
+
+        """
+        if not self.run_id:
+            raise ValueError("No run has been started")
+        return self.check_run_by_id(self.run_id)
+
+    @staticmethod
+    async def acheck_run_by_id(run_id: str, retry=0) -> AsyncGenerator[dict, None]:
+        """
+        Async version of :func:`check_run_by_id`
+
+        Check the status of a run given its ID.
+
+        This generator will produce values to show progress of the run.
+
+        Args:
+            run_id: The `run_id` produced by a `run_optimization` call
+            retry: A number used to track the number of retries to limit re-checks. *Do not* provide this value manually.
+
+        Yields:
+            Each event will be a dict, where:
+            - `"state"` is PENDING, STARTED, RETRY, FAILURE or SUCCESS
+            - `"result"` is a string describing the progress as a percentage for a PENDING state, or the error for a FAILURE state or the results for a SUCCESS state
+
+        """
+        if retry > MAX_RETRIES:
+            raise HirundoError("Max retries reached")
+        last_event = None
+        async with httpx.AsyncClient(
+            timeout=httpx.Timeout(None, connect=5.0)
+        ) as client:
+            async_iterator = await aiter_sse_retrying(
+                client,
+                "GET",
+                f"{API_HOST}/dataset-optimization/run/{run_id}",
+                headers=auth_headers,
+            )
+            async for sse in async_iterator:
+                if sse.event == "ping":
+                    continue
+                logger.debug(
+                    "[ASYNC] Received event: %s with data: %s and ID: %s and retry: %s",
+                    sse.event,
+                    sse.data,
+                    sse.id,
+                    sse.retry,
+                )
+                last_event = json.loads(sse.data)
+                yield last_event["data"]
+        if not last_event or last_event["data"]["state"] == "PENDING":
+            OptimizationDataset.acheck_run_by_id(run_id, retry + 1)
+
+    async def acheck_run(self) -> AsyncGenerator[dict, None]:
+        """
+        Async version of :func:`check_run`
+
+        Check the status of the current active instance's run.
+
+        This generator will produce values to show progress of the run.
+
+        Yields:
+            Each event will be a dict, where:
+            - `"state"` is PENDING, STARTED, RETRY, FAILURE or SUCCESS
+            - `"result"` is a string describing the progress as a percentage for a PENDING state, or the error for a FAILURE state or the results for a SUCCESS state
+
+        """
+        if not self.run_id:
+            raise ValueError("No run has been started")
+        async for iteration in self.acheck_run_by_id(self.run_id):
+            yield iteration
+
+    @staticmethod
+    def cancel_by_id(run_id: str) -> None:
+        """
+        Cancel the dataset optimization run for the given `run_id`.
+
+        Args:
+            run_id: The ID of the run to cancel
+        """
+        if not run_id:
+            raise ValueError("No run has been started")
+        response = requests.delete(
+            f"{API_HOST}/dataset-optimization/run/{run_id}",
+            headers=auth_headers,
+            timeout=MODIFY_TIMEOUT,
+        )
+        response.raise_for_status()
+
+    def cancel(self) -> None:
+        """
+        Cancel the current active instance's run.
+        """
+        if not self.run_id:
+            raise ValueError("No run has been started")
+        self.cancel_by_id(self.run_id)

hirundo/enum.py

@@ -0,0 +1,20 @@
+from enum import Enum
+
+
+class LabellingType(str, Enum):
+    """
+    Enum indicate what type of labelling is used for the given dataset.
+    Supported types are:
+    """
+
+    SingleLabelClassification = "SingleLabelClassification"
+    ObjectDetection = "ObjectDetection"
+
+
+class DatasetMetadataType(str, Enum):
+    """
+    Enum indicate what type of metadata is provided for the given dataset.
+    Supported types are:
+    """
+
+    HirundoCSV = "HirundoCSV"

hirundo/git.py

@@ -0,0 +1,158 @@
+import logging
+import re
+from typing import Annotated, Union
+
+import pydantic
+import requests
+from pydantic import BaseModel, field_validator
+from pydantic_core import Url
+
+from hirundo._env import API_HOST
+from hirundo._headers import auth_headers, json_headers
+from hirundo._timeouts import MODIFY_TIMEOUT, READ_TIMEOUT
+
+logger = logging.getLogger(__name__)
+
+
+class GitPlainAuthBase(BaseModel):
+    username: str
+    """
+    The username for the Git repository
+    """
+    password: str
+    """
+    The password for the Git repository
+    """
+
+
+class GitSSHAuthBase(BaseModel):
+    ssh_key: str
+    """
+    The SSH key for the Git repository
+    """
+    ssh_password: Union[str, None]
+    """
+    The password for the SSH key for the Git repository.
+    """
+
+
+class GitRepo(BaseModel):
+    id: Union[int, None] = None
+    """
+    The ID of the Git repository.
+    """
+
+    name: str
+    """
+    A name to identify the Git repository in the Hirundo system.
+    """
+    repository_url: Annotated[str, Url]
+    """
+    The URL of the Git repository, it should start with `ssh://` or `https://` or be in the form `user@host:path`.
+    If it is in the form `user@host:path`, it will be rewritten to `ssh://user@host:path`.
+    """
+    organization_id: Union[int, None] = None
+    """
+    The ID of the organization that the Git repository belongs to.
+    If not provided, it will be assigned to your default organization.
+    """
+
+    plain_auth: Union[GitPlainAuthBase, None] = pydantic.Field(
+        default=None, examples=[None, {"username": "ben", "password": "password"}]
+    )
+    """
+    The plain authentication details for the Git repository.
+    Use this if using a special user with a username and password for authentication.
+    """
+    ssh_auth: Union[GitSSHAuthBase, None] = pydantic.Field(
+        default=None,
+        examples=[
+            {
+                "ssh_key": "SOME_PRIVATE_SSH_KEY",
+                "ssh_password": "SOME_SSH_KEY_PASSWORD",
+            },
+            None,
+        ],
+    )
+    """
+    The SSH authentication details for the Git repository.
+    Use this if using an SSH key for authentication.
+    Optionally, you can provide a password for the SSH key.
+    """
+
+    @field_validator("repository_url", mode="before", check_fields=True)
+    @classmethod
+    def check_valid_repository_url(cls, repository_url: str):
+        # Check if the URL already has a protocol
+        if not re.match(r"^[a-z]+://", repository_url):
+            # Check if the URL has the `@` and `:` pattern with a non-numeric section before the next slash
+            match = re.match(r"([^@]+@[^:]+):([^0-9/][^/]*)/(.+)", repository_url)
+            if match:
+                user_host = match.group(1)
+                path = match.group(2) + "/" + match.group(3)
+                rewritten_url = f"ssh://{user_host}/{path}"
+                logger.info("Modified Git repo to add SSH protocol", rewritten_url)
+                return rewritten_url
+        if not repository_url.startswith("ssh://") and not repository_url.startswith(
+            "https://"
+        ):
+            raise ValueError("Repository URL must start with 'ssh://' or 'https://'")
+        return repository_url
+
+    def create(self):
+        """
+        Create a Git repository in the Hirundo system.
+        """
+        git_repo = requests.post(
+            f"{API_HOST}/git-repo/",
+            json=self.model_dump(),
+            headers={
+                **json_headers,
+                **auth_headers,
+            },
+            timeout=MODIFY_TIMEOUT,
+        )
+        git_repo.raise_for_status()
+        git_repo_id = git_repo.json()["id"]
+        self.id = git_repo_id
+        return git_repo_id
+
+    @staticmethod
+    def list():
+        """
+        List all Git repositories in the Hirundo system.
+        """
+        git_repos = requests.get(
+            f"{API_HOST}/git-repo/",
+            headers={
+                **auth_headers,
+            },
+            timeout=READ_TIMEOUT,
+        )
+        git_repos.raise_for_status()
+        return git_repos.json()
+
+    @staticmethod
+    def delete_by_id(git_repo_id: int):
+        """
+        Delete a Git repository by its ID.
+
+        Args:
+            git_repo_id: The ID of the Git repository to delete
+        """
+        git_repo = requests.delete(
+            f"{API_HOST}/git-repo/{git_repo_id}",
+            headers={
+                **auth_headers,
+            },
+            timeout=MODIFY_TIMEOUT,
+        )
+        git_repo.raise_for_status()
+
+    def delete(self):
+        """
+        Delete the Git repository created by this instance.
+        """
+        if not self.id:
+            raise ValueError("No GitRepo has been created")
+        GitRepo.delete_by_id(self.id)

hirundo/storage.py

@@ -0,0 +1,260 @@
+import typing
+from enum import Enum
+from typing import Union
+
+import pydantic
+import requests
+from pydantic import BaseModel, model_validator
+from pydantic_core import Url
+
+from hirundo._constraints import S3BucketUrl, StorageIntegrationName
+from hirundo._env import API_HOST
+from hirundo._headers import auth_headers, json_headers
+from hirundo._timeouts import MODIFY_TIMEOUT, READ_TIMEOUT
+from hirundo.git import GitRepo
+
+
+class StorageS3(BaseModel):
+    endpoint_url: Union[Url, None] = None
+    bucket_url: S3BucketUrl
+    region_name: str
+    # ⬆️ We could restrict this, but if we're allowing custom endpoints then the validation may be wrong
+    access_key_id: Union[str, None] = None
+    secret_access_key: Union[str, None] = None
+
+
+class StorageGCP(BaseModel):
+    bucket_name: str
+    project: str
+    credentials_json: Union[dict, None] = None
+
+
+# TODO: Azure storage integration is coming soon
+# class StorageAzure(BaseModel):
+#     container: str
+#     account_name: str
+#     account_key: str
+
+
+class StorageGit(BaseModel):
+    repo_id: Union[int, None] = None
+    """
+    The ID of the Git repository in the Hirundo system.
+    Either `repo_id` or `repo` must be provided.
+    """
+    repo: Union[GitRepo, None] = None
+    """
+    The Git repository to link to.
+    Either `repo_id` or `repo` must be provided.
+    """
+    branch: str
+    """
+    The branch of the Git repository to link to.
+    """
+
+    @model_validator(mode="after")
+    def validate_repo(self):
+        if self.repo_id is None and self.repo is None:
+            raise ValueError("Either repo_id or repo must be provided")
+        return self
+
+
+class StorageTypes(str, Enum):
+    """
+    Enum for the different types of storage integrations.
+    Supported types are:
+    """
+
+    S3 = "S3"
+    GCP = "GCP"
+    # AZURE = "Azure"  TODO: Azure storage integration is coming soon
+    GIT = "Git"
+
+
+class StorageIntegration(BaseModel):
+    id: Union[int, None] = None
+
+    organization_id: Union[int, None] = None
+    """
+    The ID of the organization that the `StorageIntegration` belongs to.
+    If not provided, it will be assigned to your default organization.
+    """
+
+    name: StorageIntegrationName
+    """
+    A name to identify the `StorageIntegration` in the Hirundo system.
+    """
+    type: StorageTypes = pydantic.Field(
+        examples=[
+            StorageTypes.S3,
+            StorageTypes.GCP,
+            # StorageTypes.AZURE,  TODO: Azure storage integration is coming soon
+            StorageTypes.GIT,
+        ]
+    )
+    """
+    The type of the `StorageIntegration`.
+    Supported types are:
+    - `S3`
+    - `GCP`
+    - `Azure` (coming soon)
+    - `Git`
+    """
+    s3: Union[StorageS3, None] = pydantic.Field(
+        default=None,
+        examples=[
+            {
+                "bucket_url": "s3://my-bucket",
+                "region_name": "us-west-2",
+                "access_key_id": "my-access-key",
+                "secret_access_key": "REDACTED",
+            },
+            None,
+            None,
+            None,
+        ],
+    )
+    """
+    The Amazon Web Services (AWS) S3 storage integration details.
+    Use this if you want to link to an S3 bucket.
+    """
+    gcp: Union[StorageGCP, None] = pydantic.Field(
+        default=None,
+        examples=[
+            None,
+            {
+                "bucket_name": "my-bucket",
+                "project": "my-project",
+                "credentials_json": {
+                    "type": "service_account",
+                    "project_id": "my-project",
+                    "private_key_id": "my-key-id",
+                    "private_key": "REDACTED",
+                    "client_email": "my-service-account@my-project.iam.gserviceaccount.com",
+                    "client_id": "my-id",
+                    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                    "token_uri": "https://oauth2.googleapis.com/token",
+                    "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+                    "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/my-service-account%40my-project.iam.gserviceaccount.com",
+                    "universe_domain": "googleapis.com",
+                },
+            },
+            None,
+            None,
+        ],
+    )
+    """
+    The Google Cloud (GCP) Storage integration details.
+    Use this if you want to link to an GCS bucket.
+    """
+    azure: None = None
+    # azure: Union[StorageAzure, None] = pydantic.Field(
+    #     default=None,
+    #     examples=[
+    #         None,
+    #         None,
+    #         {
+    #             "container": "my-container",
+    #             "account_name": "my-account-name",
+    #             "account_key": "my-account",
+    #         },
+    #         None,
+    #     ],
+    # )  TODO: Azure storage integration is coming soon
+    git: Union[StorageGit, None] = pydantic.Field(
+        default=None,
+        examples=[
+            None,
+            None,
+            None,
+            {
+                "repo_id": "my-repo-id",
+                "repo": {
+                    "name": "test-dataset",
+                    "repository_url": "https://github.com/Hirundo-io/test-dataset.git",
+                },
+                "branch": "main",
+                "path": "/my-path/to/dataset",
+            },
+        ],
+    )
+    """
+    The Git storage integration details.
+    Use this if you want to link to a Git repository.
+    """
+
+    @staticmethod
+    def list(organization_id: typing.Union[int, None] = None) -> list[dict]:
+        """
+        Lists all the `StorageIntegration`'s created by user's default organization
+        Note: The return type is `list[dict]` and not `list[StorageIntegration]`
+
+        Args:
+            organization_id: The ID of the organization to list `StorageIntegration`'s for.
+            If not provided, it will list `StorageIntegration`'s for the default organization.
+        """
+        storage_integrations = requests.get(
+            f"{API_HOST}/storage-integration/",
+            params={"storage_integration_organization_id": organization_id},
+            headers=auth_headers,
+            timeout=READ_TIMEOUT,
+        )
+        storage_integrations.raise_for_status()
+        return storage_integrations.json()
+
+    @staticmethod
+    def delete_by_id(storage_integration_id) -> None:
+        """
+        Deletes a `StorageIntegration` instance from the server by its ID
+
+        Args:
+            storage_integration_id: The ID of the `StorageIntegration` to delete
+        """
+        storage_integration = requests.delete(
+            f"{API_HOST}/storage-integration/{storage_integration_id}",
+            headers=auth_headers,
+            timeout=MODIFY_TIMEOUT,
+        )
+        storage_integration.raise_for_status()
+
+    def delete(self) -> None:
+        """
+        Deletes the `StorageIntegration` instance from the server
+        """
+        if not self.id:
+            raise ValueError("No StorageIntegration has been created")
+        self.delete_by_id(self.id)
+
+    def create(self) -> int:
+        """
+        Create a `StorageIntegration` instance on the server
+        """
+        if self.git and self.git.repo:
+            self.git.repo_id = self.git.repo.create()
+        storage_integration = requests.post(
+            f"{API_HOST}/storage-integration/",
+            json=self.model_dump(),
+            headers={
+                **json_headers,
+                **auth_headers,
+            },
+            timeout=MODIFY_TIMEOUT,
+        )
+        storage_integration.raise_for_status()
+        storage_integration_id = storage_integration.json()["id"]
+        self.id = storage_integration_id
+        return storage_integration_id
+
+
+class StorageLink(BaseModel):
+    storage_integration: StorageIntegration
+    """
+    The `StorageIntegration` instance to link to.
+    """
+    path: str = "/"
+    """
+    Path for the `root` to link to within the `StorageIntegration` instance,
+    e.g. a prefix path/folder within an S3 Bucket / GCP Bucket / Azure Blob storage / Git repo.
+
+    Note: Only files in this path will be retrieved and it will be used as the root for paths in the CSV.
+    """

hirundo-0.1.3.dist-info/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2024, Hirundo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

hirundo-0.1.3.dist-info/METADATA

@@ -0,0 +1,114 @@
+Metadata-Version: 2.1
+Name: hirundo
+Version: 0.1.3
+Summary: This package is used to interface with Hirundo's platform. It provides a simple API to optimize your ML datasets.
+Author-email: Hirundo <dev@hirundo.io>
+License: MIT License
+        
+        Copyright (c) 2024, Hirundo
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/Hirundo-io/hirundo-client
+Keywords: dataset,machine learning,data science,data engineering
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyyaml >=6.0.1
+Requires-Dist: types-PyYAML >=6.0.12
+Requires-Dist: pydantic >=2.7.1
+Requires-Dist: twine >=5.0.0
+Requires-Dist: python-dotenv >=1.0.1
+Requires-Dist: types-requests >=2.31.0
+Requires-Dist: typer >=0.12.3
+Requires-Dist: httpx >=0.27.0
+Requires-Dist: stamina >=24.2.0
+Requires-Dist: httpx-sse >=0.4.0
+Provides-Extra: dev
+Requires-Dist: pyyaml >=6.0.1 ; extra == 'dev'
+Requires-Dist: types-PyYAML >=6.0.12 ; extra == 'dev'
+Requires-Dist: pydantic >=2.7.1 ; extra == 'dev'
+Requires-Dist: twine >=5.0.0 ; extra == 'dev'
+Requires-Dist: python-dotenv >=1.0.1 ; extra == 'dev'
+Requires-Dist: types-requests >=2.31.0 ; extra == 'dev'
+Requires-Dist: types-setuptools >=69.5.0 ; extra == 'dev'
+Requires-Dist: typer >=0.12.3 ; extra == 'dev'
+Requires-Dist: httpx >=0.27.0 ; extra == 'dev'
+Requires-Dist: stamina >=24.2.0 ; extra == 'dev'
+Requires-Dist: httpx-sse >=0.4.0 ; extra == 'dev'
+Requires-Dist: pytest >=8.2.0 ; extra == 'dev'
+Requires-Dist: pytest-asyncio >=0.23.6 ; extra == 'dev'
+Requires-Dist: uv ; extra == 'dev'
+Requires-Dist: pre-commit >=3.7.1 ; extra == 'dev'
+Requires-Dist: ruff ; extra == 'dev'
+Requires-Dist: bumpver ; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: sphinx >=7.4.7 ; extra == 'docs'
+Requires-Dist: sphinx-autobuild >=2024.4.16 ; extra == 'docs'
+Requires-Dist: sphinx-click >=5.0.1 ; extra == 'docs'
+Requires-Dist: autodoc-pydantic >=2.2.0 ; extra == 'docs'
+Requires-Dist: furo ; extra == 'docs'
+Requires-Dist: sphinx-multiversion ; extra == 'docs'
+
+# Hirundo client
+
+This repo contains the source code for the Hirundo client library
+
+## Usage:
+
+To learn about how to use this library, please visit the [http://docs.hirundo.io/](documentation) or see the Google Colab examples.
+
+## Development:
+
+### Install dev dependencies
+
+```bash
+pip install -r dev-requirements.txt
+```
+
+Note: You can install and use `uv` as a faster drop-in replacement for `pip`. We have it as part of our dev dependencies for this reason.
+
+### Install `git` hooks (optional)
+
+```bash
+pre-commit install
+```
+
+### Check lint and apply formatting with Ruff (optional; pre-commit hooks run this automatically)
+
+```bash
+ruff check
+ruff format
+```
+
+### Change packages
+
+#### Update `requirements.txt` files
+
+```bash
+uv pip compile pyproject.toml
+uv pip compile --extra dev -o dev-requirements.txt -c requirements.txt pyproject.toml
+uv pip compile --extra docs -o docs-requirements.txt -c requirements.txt pyproject.toml
+```
+
+#### Sync installed packages
+
+```bash
+uv pip sync dev-requirements.txt
+```
+
+### Build process
+
+To build the package, run:
+`python -m build`
+
+### Publish documentation & releases
+
+Documentation & releases are published via GitHub Actions on merges to `main`.

hirundo-0.1.3.dist-info/RECORD

@@ -0,0 +1,18 @@
+hirundo/__init__.py,sha256=hkVyX9WRcnGZ8xis2hrGBD9WGeaYW0CAvAHLzGG1LbU,707
+hirundo/__main__.py,sha256=wcCrL4PjG51r5wVKqJhcoJPTLfHW0wNbD31DrUN0MWI,28
+hirundo/_constraints.py,sha256=-RAUV9GnCsaT9pLGSqYglKOeK0joPBBexGTo87j5nkI,425
+hirundo/_env.py,sha256=aObkRVLo9NBZiByd2FcoLrk3m8tnswuYzP4Tnj3EE-o,268
+hirundo/_headers.py,sha256=htxHRjtD91C5D0svyk-zqhKV9LwQCEZauIa4ZTAfe5k,188
+hirundo/_iter_sse_retrying.py,sha256=WLp_lw8ycBuAxoJkkGBu4y74Ajhcu11r1X-vd5_571A,3352
+hirundo/_timeouts.py,sha256=IfX8-mrLp809-A_xSLv1DhIqZnO-Qvy4FcTtOtvqLog,42
+hirundo/cli.py,sha256=qj1Txt6lOU3V10SLtzH4uEWJ4DdkdOIEQaKn8wiJMss,3922
+hirundo/dataset_optimization.py,sha256=CLo9eclW_trDwzfr6uZlJ8JQb6XpWcKtACqSTaAF_fo,14583
+hirundo/enum.py,sha256=-3w09g-_yRYIMiM8VA_Nb07WoQXf5IjyERTGonzNDs0,457
+hirundo/git.py,sha256=GtowxPL78KleVhSY3QISu7-cUPrFbWC4YWBAuzuzryw,4731
+hirundo/storage.py,sha256=CxRdSnZGf4mtzNV2Ge_hwowd9pDP7NT9-xvWTbl187M,8185
+hirundo-0.1.3.dist-info/LICENSE,sha256=fusGGjqT2RGlU6kbkaOk7d-gDnsjk17wq67AO0mwBZI,1065
+hirundo-0.1.3.dist-info/METADATA,sha256=Y_HZhL5-rlUW1KZlymYNqM2M62k_gKxewAJnH0tkB3M,4428
+hirundo-0.1.3.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+hirundo-0.1.3.dist-info/entry_points.txt,sha256=4ZtnA_Nl1Af8fLnHp3lwjbGDEGU1S6ujb_JwtuQ7ZPM,44
+hirundo-0.1.3.dist-info/top_level.txt,sha256=cmyNqrNZOAYxnywJGFI1AJBLe4SkH8HGsfFx6ncdrbI,8
+hirundo-0.1.3.dist-info/RECORD,,

hirundo-0.1.3.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (72.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

hirundo-0.1.3.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+hirundo = hirundo.cli:app

hirundo-0.1.3.dist-info/top_level.txt

@@ -0,0 +1 @@
+hirundo