hirundo 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

hirundo/__init__.py

@@ -1,35 +1,43 @@
 from .dataset_optimization import (
+    COCO,
+    YOLO,
+    HirundoCSV,
     HirundoError,
     OptimizationDataset,
+    RunArgs,
+    VisionRunArgs,
 )
 from .enum import (
     DatasetMetadataType,
-    LabellingType,
+    LabelingType,
 )
 from .git import GitRepo
 from .storage import (
+    StorageConfig,
     StorageGCP,
-    # StorageAzure,  TODO: Azure storage integration is coming soon
+    # StorageAzure,  TODO: Azure storage is coming soon
     StorageGit,
-    StorageIntegration,
-    StorageLink,
     StorageS3,
     StorageTypes,
 )
 
 __all__ = [
+    "COCO",
+    "YOLO",
+    "HirundoCSV",
     "HirundoError",
     "OptimizationDataset",
-    "LabellingType",
+    "RunArgs",
+    "VisionRunArgs",
+    "LabelingType",
     "DatasetMetadataType",
     "GitRepo",
-    "StorageLink",
     "StorageTypes",
     "StorageS3",
     "StorageGCP",
-    # "StorageAzure",  TODO: Azure storage integration is coming soon
+    # "StorageAzure",  TODO: Azure storage is coming soon
     "StorageGit",
-    "StorageIntegration",
+    "StorageConfig",
 ]
 
-__version__ = "0.1.7"
+__version__ = "0.1.9"

hirundo/_constraints.py

@@ -1,6 +1,7 @@
 from typing import Annotated
 
-from pydantic import StringConstraints
+from pydantic import StringConstraints, UrlConstraints
+from pydantic_core import Url
 
 S3BucketUrl = Annotated[
     str,
@@ -11,7 +12,7 @@ S3BucketUrl = Annotated[
     ),
 ]
 
-StorageIntegrationName = Annotated[
+StorageConfigName = Annotated[
     str,
     StringConstraints(
         min_length=1,
@@ -19,3 +20,34 @@ StorageIntegrationName = Annotated[
         pattern=r"^[a-zA-Z0-9-_]+$",
     ),
 ]
+
+S3_MIN_LENGTH = 8
+S3_MAX_LENGTH = 1023
+S3_PATTERN = r"s3://[a-zA-Z0-9.-]{3,64}/[a-zA-Z0-9.-/]+"
+GCP_MIN_LENGTH = 8
+GCP_MAX_LENGTH = 1023
+GCP_PATTERN = r"gs://[a-zA-Z0-9.-]{3,64}/[a-zA-Z0-9.-/]+"
+
+RepoUrl = Annotated[
+    Url,
+    UrlConstraints(
+        allowed_schemes=[
+            "ssh",
+            "https",
+            "http",
+        ]
+    ),
+]
+HirundoUrl = Annotated[
+    Url,
+    UrlConstraints(
+        allowed_schemes=[
+            "file",
+            "https",
+            "http",
+            "s3",
+            "gs",
+            "ssh",
+        ]
+    ),
+]

hirundo/_env.py

@@ -1,8 +1,19 @@
+import enum
 import os
+from pathlib import Path
 
 from dotenv import load_dotenv
 
-load_dotenv()
+
+class EnvLocation(enum.Enum):
+    DOTENV = Path.cwd() / ".env"
+    HOME = Path.home() / ".hirundo.conf"
+
+
+if os.path.exists(EnvLocation.DOTENV.value):
+    load_dotenv(EnvLocation.DOTENV.value)
+elif os.path.exists(EnvLocation.HOME.value):
+    load_dotenv(EnvLocation.HOME.value)
 
 API_HOST = os.getenv("API_HOST", "https://api.hirundo.io")
 API_KEY = os.getenv("API_KEY")

hirundo/_http.py

@@ -0,0 +1,19 @@
+from requests import Response
+
+import hirundo.logger
+
+logger = hirundo.logger.get_logger(__name__)
+
+MINIMUM_CLIENT_SERVER_ERROR_CODE = 400
+
+
+def raise_for_status_with_reason(response: Response):
+    try:
+        if response.status_code >= MINIMUM_CLIENT_SERVER_ERROR_CODE:
+            response.reason = response.json().get("reason", None)
+            if response.reason is None:
+                response.reason = response.json().get("detail", None)
+    except Exception as e:
+        logger.debug("Could not parse response as JSON: %s", e)
+
+    response.raise_for_status()

hirundo/_iter_sse_retrying.py

@@ -1,19 +1,27 @@
 import asyncio
 import time
+import typing
+import uuid
 from collections.abc import AsyncGenerator, Generator
-from typing import Union
 
 import httpx
-from httpx_sse import ServerSentEvent, aconnect_sse, connect_sse
+import requests
+import urllib3
+from httpx_sse import ServerSentEvent, SSEError, aconnect_sse, connect_sse
 from stamina import retry
 
+from hirundo._timeouts import READ_TIMEOUT
+from hirundo.logger import get_logger
+
+logger = get_logger(__name__)
+
 
 # Credit: https://github.com/florimondmanca/httpx-sse/blob/master/README.md#handling-reconnections
 def iter_sse_retrying(
     client: httpx.Client,
     method: str,
     url: str,
-    headers: Union[dict[str, str], None] = None,
+    headers: typing.Optional[dict[str, str]] = None,
 ) -> Generator[ServerSentEvent, None, None]:
     if headers is None:
         headers = {}
@@ -28,7 +36,13 @@ def iter_sse_retrying(
     #  This may happen when the server is overloaded and closes the connection or
     #  when Kubernetes restarts / replaces a pod.
     #  Likewise, this will likely be temporary, hence the retries.
-    @retry(on=(httpx.ReadError, httpx.RemoteProtocolError))
+    @retry(
+        on=(
+            httpx.ReadError,
+            httpx.RemoteProtocolError,
+            urllib3.exceptions.ReadTimeoutError,
+        )
+    )
     def _iter_sse():
         nonlocal last_event_id, reconnection_delay
 
@@ -44,13 +58,27 @@ def iter_sse_retrying(
             connect_headers["Last-Event-ID"] = last_event_id
 
         with connect_sse(client, method, url, headers=connect_headers) as event_source:
-            for sse in event_source.iter_sse():
-                last_event_id = sse.id
-
-                if sse.retry is not None:
-                    reconnection_delay = sse.retry / 1000
-
-                yield sse
+            try:
+                for sse in event_source.iter_sse():
+                    last_event_id = sse.id
+
+                    if sse.retry is not None:
+                        reconnection_delay = sse.retry / 1000
+
+                    yield sse
+            except SSEError:
+                logger.error("SSE error occurred. Trying regular request")
+                response = requests.get(
+                    url,
+                    headers=connect_headers,
+                    timeout=READ_TIMEOUT,
+                )
+                yield ServerSentEvent(
+                    event="",
+                    data=response.text,
+                    id=uuid.uuid4().hex,
+                    retry=None,
+                )
 
     return _iter_sse()
 
@@ -72,7 +100,13 @@ async def aiter_sse_retrying(
     #  This may happen when the server is overloaded and closes the connection or
     #  when Kubernetes restarts / replaces a pod.
     #  Likewise, this will likely be temporary, hence the retries.
-    @retry(on=(httpx.ReadError, httpx.RemoteProtocolError))
+    @retry(
+        on=(
+            httpx.ReadError,
+            httpx.RemoteProtocolError,
+            urllib3.exceptions.ReadTimeoutError,
+        )
+    )
     async def _iter_sse() -> AsyncGenerator[ServerSentEvent, None]:
         nonlocal last_event_id, reconnection_delay
 
@@ -86,12 +120,22 @@ async def aiter_sse_retrying(
         async with aconnect_sse(
             client, method, url, headers=connect_headers
         ) as event_source:
-            async for sse in event_source.aiter_sse():
-                last_event_id = sse.id
-
-                if sse.retry is not None:
-                    reconnection_delay = sse.retry / 1000
-
-                yield sse
+            try:
+                async for sse in event_source.aiter_sse():
+                    last_event_id = sse.id
+
+                    if sse.retry is not None:
+                        reconnection_delay = sse.retry / 1000
+
+                    yield sse
+            except SSEError:
+                logger.error("SSE error occurred. Trying regular request")
+                response = await client.get(url, headers=connect_headers)
+                yield ServerSentEvent(
+                    event="",
+                    data=response.text,
+                    id=uuid.uuid4().hex,
+                    retry=None,
+                )
 
     return _iter_sse()

hirundo/cli.py

@@ -1,11 +1,14 @@
+import os
 import re
 import sys
+import typing
+from pathlib import Path
 from typing import Annotated
 from urllib.parse import urlparse
 
 import typer
 
-from hirundo._env import API_HOST
+from hirundo._env import API_HOST, EnvLocation
 
 docs = "sphinx" in sys.modules
 hirundo_epilog = (
@@ -23,7 +26,9 @@ app = typer.Typer(
 )
 
 
-def upsert_env(var_name: str, var_value: str):
+def _upsert_env(
+    dotenv_filepath: typing.Union[str, Path], var_name: str, var_value: str
+):
     """
     Change an environment variable in the .env file.
     If the variable does not exist, it will be added.
@@ -32,18 +37,30 @@ def upsert_env(var_name: str, var_value: str):
         var_name: The name of the environment variable to change.
         var_value: The new value of the environment variable.
     """
-    dotenv = "./.env"
     regex = re.compile(rf"^{var_name}=.*$")
-    with open(dotenv) as f:
-        lines = f.readlines()
+    lines = []
+    if os.path.exists(dotenv_filepath):
+        with open(dotenv_filepath) as f:
+            lines = f.readlines()
 
-    with open(dotenv, "w") as f:
+    with open(dotenv_filepath, "w") as f:
         f.writelines(line for line in lines if not regex.search(line) and line != "\n")
 
-    with open(dotenv, "a") as f:
+    with open(dotenv_filepath, "a") as f:
         f.writelines(f"\n{var_name}={var_value}")
 
 
+def upsert_env(var_name: str, var_value: str):
+    if os.path.exists(EnvLocation.DOTENV.value):
+        # If a `.env` file exists, re-use it
+        _upsert_env(EnvLocation.DOTENV.value, var_name, var_value)
+        return EnvLocation.DOTENV.name
+    else:
+        # Create a `.hirundo.conf` file with environment variables in the home directory
+        _upsert_env(EnvLocation.HOME.value, var_name, var_value)
+        return EnvLocation.HOME.name
+
+
 def fix_api_host(api_host: str):
     if not api_host.startswith("http") and not api_host.startswith("https"):
         api_host = f"https://{api_host}"
@@ -72,8 +89,15 @@ def setup_api_key(
     Setup the API key for the Hirundo client library.
     Values are saved to a .env file in the current directory for use by the library in requests.
     """
-    upsert_env("API_KEY", api_key)
-    print("API key saved to .env for future use. Please do not share the .env file")
+    saved_to = upsert_env("API_KEY", api_key)
+    if saved_to == EnvLocation.HOME.name:
+        print(
+            "API key saved to ~/.hirundo.conf for future use. Please do not share the ~/.hirundo.conf file since it contains your secret API key."
+        )
+    elif saved_to == EnvLocation.DOTENV.name:
+        print(
+            "API key saved to local .env file for future use. Please do not share the .env file since it contains your secret API key."
+        )
 
 
 @app.command("change-remote", epilog=hirundo_epilog)
@@ -94,8 +118,13 @@ def change_api_remote(
     """
     api_host = fix_api_host(api_host)
 
-    upsert_env("API_HOST", api_host)
-    print("API host saved to .env for future use. Please do not share this file")
+    saved_to = upsert_env("API_HOST", api_host)
+    if saved_to == EnvLocation.HOME.name:
+        print(
+            "API host saved to ~/.hirundo.conf for future use. Please do not share the ~/.hirundo.conf file"
+        )
+    elif saved_to == EnvLocation.DOTENV.name:
+        print("API host saved to .env for future use. Please do not share this file")
 
 
 @app.command("setup", epilog=hirundo_epilog)
@@ -123,11 +152,41 @@ def setup(
     Setup the Hirundo client library.
     """
     api_host = fix_api_host(api_host)
-    upsert_env("API_HOST", api_host)
-    upsert_env("API_KEY", api_key)
-    print(
-        "API host and API key saved to .env for future use. Please do not share this file"
-    )
+    api_host_saved_to = upsert_env("API_HOST", api_host)
+    api_key_saved_to = upsert_env("API_KEY", api_key)
+    if api_host_saved_to != api_key_saved_to:
+        print(
+            "API host and API key saved to different locations. This should not happen. Please report this issue."
+        )
+        if (
+            api_host_saved_to == EnvLocation.HOME.name
+            and api_key_saved_to == EnvLocation.DOTENV.name
+        ):
+            print(
+                "API host saved to ~/.hirundo.conf for future use. Please do not share the ~/.hirundo.conf file"
+            )
+            print(
+                "API key saved to local .env file for future use. Please do not share the .env file since it contains your secret API key."
+            )
+        elif (
+            api_host_saved_to == EnvLocation.DOTENV.name
+            and api_key_saved_to == EnvLocation.HOME.name
+        ):
+            print(
+                "API host saved to .env for future use. Please do not share this file"
+            )
+            print(
+                "API key saved to ~/.hirundo.conf for future use. Please do not share the ~/.hirundo.conf file since it contains your secret API key."
+            )
+        return
+    if api_host_saved_to == EnvLocation.HOME.name:
+        print(
+            "API host and API key saved to ~/.hirundo.conf for future use. Please do not share the ~/.hirundo.conf file since it contains your secret API key."
+        )
+    elif api_host_saved_to == EnvLocation.DOTENV.name:
+        print(
+            "API host and API key saved to .env for future use. Please do not share this file since it contains your secret API key."
+        )
 
 
 typer_click_object = typer.main.get_command(app)