hirundo 0.1.8__py3-none-any.whl → 0.1.16__py3-none-any.whl

hirundo/__init__.py

@@ -1,35 +1,50 @@
+from .dataset_enum import (
+    DatasetMetadataType,
+    LabelingType,
+)
 from .dataset_optimization import (
+    COCO,
+    YOLO,
+    HirundoCSV,
     HirundoError,
     OptimizationDataset,
+    RunArgs,
+    VisionRunArgs,
 )
-from .enum import (
-    DatasetMetadataType,
-    LabellingType,
-)
-from .git import GitRepo
+from .dataset_optimization_results import DatasetOptimizationResults
+from .git import GitPlainAuth, GitRepo, GitSSHAuth
 from .storage import (
+    StorageConfig,
     StorageGCP,
-    # StorageAzure,  TODO: Azure storage integration is coming soon
+    # StorageAzure,  TODO: Azure storage is coming soon
     StorageGit,
-    StorageIntegration,
-    StorageLink,
     StorageS3,
     StorageTypes,
 )
+from .unzip import load_df, load_from_zip
 
 __all__ = [
+    "COCO",
+    "YOLO",
+    "HirundoCSV",
     "HirundoError",
     "OptimizationDataset",
-    "LabellingType",
+    "RunArgs",
+    "VisionRunArgs",
+    "LabelingType",
     "DatasetMetadataType",
+    "GitPlainAuth",
     "GitRepo",
-    "StorageLink",
+    "GitSSHAuth",
     "StorageTypes",
     "StorageS3",
     "StorageGCP",
-    # "StorageAzure",  TODO: Azure storage integration is coming soon
+    # "StorageAzure",  TODO: Azure storage is coming soon
     "StorageGit",
-    "StorageIntegration",
+    "StorageConfig",
+    "DatasetOptimizationResults",
+    "load_df",
+    "load_from_zip",
 ]
 
-__version__ = "0.1.8"
+__version__ = "0.1.16"

hirundo/_constraints.py

@@ -1,6 +1,7 @@
 from typing import Annotated
 
-from pydantic import StringConstraints
+from pydantic import StringConstraints, UrlConstraints
+from pydantic_core import Url
 
 S3BucketUrl = Annotated[
     str,
@@ -11,7 +12,7 @@ S3BucketUrl = Annotated[
     ),
 ]
 
-StorageIntegrationName = Annotated[
+StorageConfigName = Annotated[
     str,
     StringConstraints(
         min_length=1,
@@ -19,3 +20,34 @@ StorageIntegrationName = Annotated[
         pattern=r"^[a-zA-Z0-9-_]+$",
     ),
 ]
+
+S3_MIN_LENGTH = 8
+S3_MAX_LENGTH = 1023
+S3_PATTERN = r"s3://[a-zA-Z0-9.-]{3,64}/[a-zA-Z0-9.-/]+"
+GCP_MIN_LENGTH = 8
+GCP_MAX_LENGTH = 1023
+GCP_PATTERN = r"gs://[a-zA-Z0-9.-]{3,64}/[a-zA-Z0-9.-/]+"
+
+RepoUrl = Annotated[
+    Url,
+    UrlConstraints(
+        allowed_schemes=[
+            "ssh",
+            "https",
+            "http",
+        ]
+    ),
+]
+HirundoUrl = Annotated[
+    Url,
+    UrlConstraints(
+        allowed_schemes=[
+            "file",
+            "https",
+            "http",
+            "s3",
+            "gs",
+            "ssh",
+        ]
+    ),
+]

hirundo/_dataframe.py

@@ -0,0 +1,43 @@
+has_pandas = False
+has_polars = False
+
+pd = None
+pl = None
+int32 = type[None]
+float32 = type[None]
+string = type[None]
+#  ⬆️ These are just placeholders for the int32, float32 and string types
+#    for when neither pandas nor polars are available
+
+try:
+    import numpy as np
+    import pandas as pd
+
+    has_pandas = True
+    int32 = np.int32
+    float32 = np.float32
+    string = str
+except ImportError:
+    pass
+
+try:
+    import polars as pl
+    import polars.datatypes as pl_datatypes
+
+    has_polars = True
+    int32 = pl_datatypes.Int32
+    float32 = pl_datatypes.Float32
+    string = pl_datatypes.String
+except ImportError:
+    pass
+
+
+__all__ = [
+    "has_polars",
+    "has_pandas",
+    "pd",
+    "pl",
+    "int32",
+    "float32",
+    "string",
+]

hirundo/_env.py

@@ -2,11 +2,11 @@ import enum
 import os
 from pathlib import Path
 
-from dotenv import load_dotenv
+from dotenv import find_dotenv, load_dotenv
 
 
 class EnvLocation(enum.Enum):
-    DOTENV = Path.cwd() / ".env"
+    DOTENV = find_dotenv(".env")
     HOME = Path.home() / ".hirundo.conf"
 
 

hirundo/_headers.py

@@ -1,13 +1,29 @@
 from hirundo._env import API_KEY, check_api_key
 
-json_headers = {
+HIRUNDO_API_VERSION = "0.2"
+
+_json_headers = {
     "Content-Type": "application/json",
     "Accept": "application/json",
 }
 
 
-def get_auth_headers():
+def _get_auth_headers():
     check_api_key()
     return {
         "Authorization": f"Bearer {API_KEY}",
     }
+
+
+def _get_api_version_header():
+    return {
+        "HIRUNDO-API-VERSION": HIRUNDO_API_VERSION,
+    }
+
+
+def get_headers():
+    return {
+        **_json_headers,
+        **_get_auth_headers(),
+        **_get_api_version_header(),
+    }

hirundo/_http.py

@@ -4,11 +4,16 @@ import hirundo.logger
 
 logger = hirundo.logger.get_logger(__name__)
 
+MINIMUM_CLIENT_SERVER_ERROR_CODE = 400
+
 
 def raise_for_status_with_reason(response: Response):
     try:
-        response.reason = response.json().get("reason", None)
+        if response.status_code >= MINIMUM_CLIENT_SERVER_ERROR_CODE:
+            response.reason = response.json().get("reason", None)
+            if response.reason is None:
+                response.reason = response.json().get("detail", None)
     except Exception as e:
-        logger.debug("Failed to parse response as JSON: %s", e)
+        logger.debug("Could not parse response as JSON: %s", e)
 
     response.raise_for_status()

hirundo/_iter_sse_retrying.py

@@ -1,12 +1,20 @@
 import asyncio
 import time
 import typing
+import uuid
 from collections.abc import AsyncGenerator, Generator
 
 import httpx
-from httpx_sse import ServerSentEvent, aconnect_sse, connect_sse
+import requests
+import urllib3
+from httpx_sse import ServerSentEvent, SSEError, aconnect_sse, connect_sse
 from stamina import retry
 
+from hirundo._timeouts import READ_TIMEOUT
+from hirundo.logger import get_logger
+
+logger = get_logger(__name__)
+
 
 # Credit: https://github.com/florimondmanca/httpx-sse/blob/master/README.md#handling-reconnections
 def iter_sse_retrying(
@@ -28,7 +36,13 @@ def iter_sse_retrying(
     #  This may happen when the server is overloaded and closes the connection or
     #  when Kubernetes restarts / replaces a pod.
     #  Likewise, this will likely be temporary, hence the retries.
-    @retry(on=(httpx.ReadError, httpx.RemoteProtocolError))
+    @retry(
+        on=(
+            httpx.ReadError,
+            httpx.RemoteProtocolError,
+            urllib3.exceptions.ReadTimeoutError,
+        )
+    )
     def _iter_sse():
         nonlocal last_event_id, reconnection_delay
 
@@ -44,13 +58,27 @@ def iter_sse_retrying(
             connect_headers["Last-Event-ID"] = last_event_id
 
         with connect_sse(client, method, url, headers=connect_headers) as event_source:
-            for sse in event_source.iter_sse():
-                last_event_id = sse.id
-
-                if sse.retry is not None:
-                    reconnection_delay = sse.retry / 1000
-
-                yield sse
+            try:
+                for sse in event_source.iter_sse():
+                    last_event_id = sse.id
+
+                    if sse.retry is not None:
+                        reconnection_delay = sse.retry / 1000
+
+                    yield sse
+            except SSEError:
+                logger.error("SSE error occurred. Trying regular request")
+                response = requests.get(
+                    url,
+                    headers=connect_headers,
+                    timeout=READ_TIMEOUT,
+                )
+                yield ServerSentEvent(
+                    event="",
+                    data=response.text,
+                    id=uuid.uuid4().hex,
+                    retry=None,
+                )
 
     return _iter_sse()
 
@@ -72,7 +100,13 @@ async def aiter_sse_retrying(
     #  This may happen when the server is overloaded and closes the connection or
     #  when Kubernetes restarts / replaces a pod.
     #  Likewise, this will likely be temporary, hence the retries.
-    @retry(on=(httpx.ReadError, httpx.RemoteProtocolError))
+    @retry(
+        on=(
+            httpx.ReadError,
+            httpx.RemoteProtocolError,
+            urllib3.exceptions.ReadTimeoutError,
+        )
+    )
     async def _iter_sse() -> AsyncGenerator[ServerSentEvent, None]:
         nonlocal last_event_id, reconnection_delay
 
@@ -86,12 +120,22 @@ async def aiter_sse_retrying(
         async with aconnect_sse(
             client, method, url, headers=connect_headers
         ) as event_source:
-            async for sse in event_source.aiter_sse():
-                last_event_id = sse.id
-
-                if sse.retry is not None:
-                    reconnection_delay = sse.retry / 1000
-
-                yield sse
+            try:
+                async for sse in event_source.aiter_sse():
+                    last_event_id = sse.id
+
+                    if sse.retry is not None:
+                        reconnection_delay = sse.retry / 1000
+
+                    yield sse
+            except SSEError:
+                logger.error("SSE error occurred. Trying regular request")
+                response = await client.get(url, headers=connect_headers)
+                yield ServerSentEvent(
+                    event="",
+                    data=response.text,
+                    id=uuid.uuid4().hex,
+                    retry=None,
+                )
 
     return _iter_sse()

hirundo/_timeouts.py

@@ -1,2 +1,3 @@
 READ_TIMEOUT = 30.0
 MODIFY_TIMEOUT = 60.0
+DOWNLOAD_READ_TIMEOUT = 600.0  # 10 minutes

hirundo/cli.py

@@ -7,6 +7,8 @@ from typing import Annotated
 from urllib.parse import urlparse
 
 import typer
+from rich.console import Console
+from rich.table import Table
 
 from hirundo._env import API_HOST, EnvLocation
 
@@ -189,6 +191,56 @@ def setup(
         )
 
 
+@app.command("check-run", epilog=hirundo_epilog)
+def check_run(
+    run_id: str,
+):
+    """
+    Check the status of a run.
+    """
+    from hirundo.dataset_optimization import OptimizationDataset
+
+    results = OptimizationDataset.check_run_by_id(run_id)
+    print(f"Run results saved to {results.cached_zip_path}")
+
+
+@app.command("list-runs", epilog=hirundo_epilog)
+def list_runs():
+    """
+    List all runs available.
+    """
+    from hirundo.dataset_optimization import OptimizationDataset
+
+    runs = OptimizationDataset.list_runs()
+
+    console = Console()
+    table = Table(
+        title="Runs:",
+        expand=True,
+    )
+    cols = (
+        "Dataset name",
+        "Run ID",
+        "Status",
+        "Created At",
+        "Run Args",
+    )
+    for col in cols:
+        table.add_column(
+            col,
+            overflow="fold",
+        )
+    for run in runs:
+        table.add_row(
+            str(run.name),
+            str(run.id),
+            str(run.status),
+            run.created_at.isoformat(),
+            run.run_args.model_dump_json() if run.run_args else None,
+        )
+    console.print(table)
+
+
 typer_click_object = typer.main.get_command(app)
 
 if __name__ == "__main__":

hirundo/dataset_enum.py

@@ -0,0 +1,23 @@
+from enum import Enum
+
+
+class LabelingType(str, Enum):
+    """
+    Enum indicate what type of labeling is used for the given dataset.
+    Supported types are:
+    """
+
+    SINGLE_LABEL_CLASSIFICATION = "SingleLabelClassification"
+    OBJECT_DETECTION = "ObjectDetection"
+    SPEECH_TO_TEXT = "SpeechToText"
+
+
+class DatasetMetadataType(str, Enum):
+    """
+    Enum indicate what type of metadata is provided for the given dataset.
+    Supported types are:
+    """
+
+    HIRUNDO_CSV = "HirundoCSV"
+    COCO = "COCO"
+    YOLO = "YOLO"