hirundo 0.1.9__py3-none-any.whl → 0.1.18__py3-none-any.whl

hirundo/__init__.py

@@ -1,43 +1,62 @@
+from .dataset_enum import (
+    DatasetMetadataType,
+    LabelingType,
+    StorageTypes,
+)
 from .dataset_optimization import (
-    COCO,
-    YOLO,
-    HirundoCSV,
     HirundoError,
     OptimizationDataset,
     RunArgs,
     VisionRunArgs,
 )
-from .enum import (
-    DatasetMetadataType,
-    LabelingType,
+from .dataset_optimization_results import DatasetOptimizationResults
+from .git import GitPlainAuth, GitRepo, GitSSHAuth
+from .labeling import (
+    COCO,
+    YOLO,
+    HirundoCSV,
+    KeylabsAuth,
+    KeylabsObjDetImages,
+    KeylabsObjDetVideo,
+    KeylabsObjSegImages,
+    KeylabsObjSegVideo,
 )
-from .git import GitRepo
 from .storage import (
     StorageConfig,
     StorageGCP,
     # StorageAzure,  TODO: Azure storage is coming soon
     StorageGit,
     StorageS3,
-    StorageTypes,
 )
+from .unzip import load_df, load_from_zip
 
 __all__ = [
     "COCO",
     "YOLO",
-    "HirundoCSV",
     "HirundoError",
+    "HirundoCSV",
+    "KeylabsAuth",
+    "KeylabsObjDetImages",
+    "KeylabsObjDetVideo",
+    "KeylabsObjSegImages",
+    "KeylabsObjSegVideo",
     "OptimizationDataset",
     "RunArgs",
     "VisionRunArgs",
-    "LabelingType",
     "DatasetMetadataType",
+    "LabelingType",
+    "GitPlainAuth",
     "GitRepo",
+    "GitSSHAuth",
     "StorageTypes",
     "StorageS3",
     "StorageGCP",
     # "StorageAzure",  TODO: Azure storage is coming soon
     "StorageGit",
     "StorageConfig",
+    "DatasetOptimizationResults",
+    "load_df",
+    "load_from_zip",
 ]
 
-__version__ = "0.1.9"
+__version__ = "0.1.18"

hirundo/_constraints.py

@@ -1,53 +1,164 @@
-from typing import Annotated
-
-from pydantic import StringConstraints, UrlConstraints
-from pydantic_core import Url
-
-S3BucketUrl = Annotated[
-    str,
-    StringConstraints(
-        min_length=8,
-        max_length=1023,
-        pattern=r"s3?://[a-z0-9.-]{3,64}[/]?",  # Only allow real S3 bucket URLs
-    ),
-]
-
-StorageConfigName = Annotated[
-    str,
-    StringConstraints(
-        min_length=1,
-        max_length=255,
-        pattern=r"^[a-zA-Z0-9-_]+$",
-    ),
-]
-
-S3_MIN_LENGTH = 8
-S3_MAX_LENGTH = 1023
-S3_PATTERN = r"s3://[a-zA-Z0-9.-]{3,64}/[a-zA-Z0-9.-/]+"
-GCP_MIN_LENGTH = 8
-GCP_MAX_LENGTH = 1023
-GCP_PATTERN = r"gs://[a-zA-Z0-9.-]{3,64}/[a-zA-Z0-9.-/]+"
-
-RepoUrl = Annotated[
-    Url,
-    UrlConstraints(
-        allowed_schemes=[
-            "ssh",
-            "https",
-            "http",
-        ]
-    ),
-]
-HirundoUrl = Annotated[
-    Url,
-    UrlConstraints(
-        allowed_schemes=[
-            "file",
-            "https",
-            "http",
-            "s3",
-            "gs",
-            "ssh",
-        ]
-    ),
-]
+import re
+import typing
+from typing import TYPE_CHECKING
+
+from hirundo._urls import (
+    LENGTH_CONSTRAINTS,
+    STORAGE_PATTERNS,
+)
+from hirundo.dataset_enum import DatasetMetadataType, LabelingType, StorageTypes
+from hirundo.labeling import COCO, YOLO, HirundoCSV, Keylabs
+
+if TYPE_CHECKING:
+    from hirundo._urls import HirundoUrl
+    from hirundo.dataset_optimization import LabelingInfo
+    from hirundo.storage import (
+        ResponseStorageConfig,
+        StorageConfig,
+        StorageGCP,
+        StorageGCPOut,
+        StorageS3,
+        StorageS3Out,
+    )
+
+LABELING_TYPES_TO_DATASET_METADATA_TYPES = {
+    LabelingType.SINGLE_LABEL_CLASSIFICATION: [
+        DatasetMetadataType.HIRUNDO_CSV,
+    ],
+    LabelingType.OBJECT_DETECTION: [
+        DatasetMetadataType.HIRUNDO_CSV,
+        DatasetMetadataType.COCO,
+        DatasetMetadataType.YOLO,
+        DatasetMetadataType.KeylabsObjDetImages,
+        DatasetMetadataType.KeylabsObjDetVideo,
+    ],
+    LabelingType.OBJECT_SEGMENTATION: [
+        DatasetMetadataType.HIRUNDO_CSV,
+        DatasetMetadataType.KeylabsObjSegImages,
+        DatasetMetadataType.KeylabsObjSegVideo,
+    ],
+    LabelingType.SEMANTIC_SEGMENTATION: [
+        DatasetMetadataType.HIRUNDO_CSV,
+    ],
+    LabelingType.PANOPTIC_SEGMENTATION: [
+        DatasetMetadataType.HIRUNDO_CSV,
+    ],
+    LabelingType.SPEECH_TO_TEXT: [
+        DatasetMetadataType.HIRUNDO_CSV,
+    ],
+}
+
+
+def validate_s3_url(str_url: str, s3_config: "StorageS3 | StorageS3Out"):
+    if (
+        len(str_url) < LENGTH_CONSTRAINTS[StorageTypes.S3]["min_length"]
+        or len(str_url) > LENGTH_CONSTRAINTS[StorageTypes.S3]["max_length"]
+    ):
+        raise ValueError("S3 URL must be between 8 and 1023 characters")
+    elif not re.match(STORAGE_PATTERNS[StorageTypes.S3], str_url):
+        raise ValueError(
+            f"Invalid S3 URL. Pattern must match: {STORAGE_PATTERNS[StorageTypes.S3]}"
+        )
+    elif not str_url.startswith(f"{s3_config.bucket_url}/"):
+        raise ValueError(f"S3 URL must start with {s3_config.bucket_url}/")
+
+
+def validate_gcp_url(str_url: str, gcp_config: "StorageGCP | StorageGCPOut"):
+    matches = re.match(STORAGE_PATTERNS[StorageTypes.GCP], str_url)
+    if (
+        len(str_url) < LENGTH_CONSTRAINTS[StorageTypes.GCP]["min_length"]
+        or len(str_url) > LENGTH_CONSTRAINTS[StorageTypes.GCP]["max_length"]
+    ):
+        raise ValueError(
+            f"GCP URL must be between {LENGTH_CONSTRAINTS[StorageTypes.GCP]['min_length']}"
+            + f" and {LENGTH_CONSTRAINTS[StorageTypes.GCP]['max_length']} characters"
+        )
+    elif not matches:
+        raise ValueError(
+            f"Invalid GCP URL. Pattern must match: {STORAGE_PATTERNS[StorageTypes.GCP]}"
+        )
+    elif (
+        matches
+        and len(matches.group(1))
+        > LENGTH_CONSTRAINTS[StorageTypes.GCP]["bucket_max_length"]
+    ):
+        raise ValueError(
+            f"GCP bucket name must be between {LENGTH_CONSTRAINTS[StorageTypes.GCP]['bucket_min_length']} "
+            + f"and {LENGTH_CONSTRAINTS[StorageTypes.GCP]['bucket_max_length']} characters"
+        )
+    elif not str_url.startswith(f"gs://{gcp_config.bucket_name}/"):
+        raise ValueError(f"GCP URL must start with gs://{gcp_config.bucket_name}")
+
+
+def validate_url(
+    url: "HirundoUrl",
+    storage_config: "StorageConfig | ResponseStorageConfig",
+) -> "HirundoUrl":
+    s3_config = storage_config.s3
+    gcp_config = storage_config.gcp
+    git_config = storage_config.git
+    str_url = str(url)
+
+    if s3_config is not None:
+        validate_s3_url(str_url, s3_config)
+    elif gcp_config is not None:
+        validate_gcp_url(str_url, gcp_config)
+    elif (
+        git_config is not None
+        and not str_url.startswith("https://")
+        and not str_url.startswith("ssh://")
+    ):
+        raise ValueError("Git URL must start with https:// or ssh://")
+    elif storage_config.type == StorageTypes.LOCAL and not str_url.startswith(
+        "file:///datasets/"
+    ):
+        raise ValueError("Local URL must start with file:///datasets/")
+    return url
+
+
+def validate_labeling_type(
+    labeling_type: "LabelingType", labeling_info: "LabelingInfo"
+) -> None:
+    """
+    Validate that the labeling type is compatible with the labeling info
+
+    Args:
+        labeling_type: The type of labeling that will be performed
+        labeling_info: The labeling info to validate
+    """
+    dataset_metadata_types = LABELING_TYPES_TO_DATASET_METADATA_TYPES[labeling_type]
+    if labeling_info.type not in dataset_metadata_types:
+        raise ValueError(
+            f"Cannot use {labeling_info.type.name} labeling info with {labeling_type.name} datasets"
+        )
+
+
+def validate_labeling_info(
+    labeling_type: "LabelingType",
+    labeling_info: "typing.Union[LabelingInfo, list[LabelingInfo]]",
+    storage_config: "typing.Union[StorageConfig, ResponseStorageConfig]",
+) -> None:
+    """
+    Validate the labeling info for a dataset
+
+    Args:
+        labeling_type: The type of labeling that will be performed
+        labeling_info: The labeling info to validate
+        storage_config: The storage configuration for the dataset.
+            StorageConfig is used to validate the URLs in the labeling info
+    """
+    if isinstance(labeling_info, list):
+        for labeling in labeling_info:
+            validate_labeling_info(labeling_type, labeling, storage_config)
+        return
+    elif isinstance(labeling_info, HirundoCSV):
+        validate_url(labeling_info.csv_url, storage_config)
+    elif isinstance(labeling_info, COCO):
+        validate_url(labeling_info.json_url, storage_config)
+    elif isinstance(labeling_info, YOLO):
+        validate_url(labeling_info.labels_dir_url, storage_config)
+        if labeling_info.data_yaml_url is not None:
+            validate_url(labeling_info.data_yaml_url, storage_config)
+    elif isinstance(labeling_info, Keylabs):
+        validate_url(labeling_info.labels_dir_url, storage_config)
+    validate_labeling_type(labeling_type, labeling_info)

hirundo/_dataframe.py

@@ -0,0 +1,43 @@
+has_pandas = False
+has_polars = False
+
+pd = None
+pl = None
+int32 = type[None]
+float32 = type[None]
+string = type[None]
+#  ⬆️ These are just placeholders for the int32, float32 and string types
+#    for when neither pandas nor polars are available
+
+try:
+    import numpy as np
+    import pandas as pd
+
+    has_pandas = True
+    int32 = np.int32
+    float32 = np.float32
+    string = str
+except ImportError:
+    pass
+
+try:
+    import polars as pl
+    import polars.datatypes as pl_datatypes
+
+    has_polars = True
+    int32 = pl_datatypes.Int32
+    float32 = pl_datatypes.Float32
+    string = pl_datatypes.String
+except ImportError:
+    pass
+
+
+__all__ = [
+    "has_polars",
+    "has_pandas",
+    "pd",
+    "pl",
+    "int32",
+    "float32",
+    "string",
+]

hirundo/_env.py

@@ -2,11 +2,11 @@ import enum
 import os
 from pathlib import Path
 
-from dotenv import load_dotenv
+from dotenv import find_dotenv, load_dotenv
 
 
 class EnvLocation(enum.Enum):
-    DOTENV = Path.cwd() / ".env"
+    DOTENV = find_dotenv(".env")
     HOME = Path.home() / ".hirundo.conf"
 
 

hirundo/_headers.py

@@ -1,13 +1,29 @@
 from hirundo._env import API_KEY, check_api_key
 
-json_headers = {
+HIRUNDO_API_VERSION = "0.2"
+
+_json_headers = {
     "Content-Type": "application/json",
     "Accept": "application/json",
 }
 
 
-def get_auth_headers():
+def _get_auth_headers():
     check_api_key()
     return {
         "Authorization": f"Bearer {API_KEY}",
     }
+
+
+def _get_api_version_header():
+    return {
+        "HIRUNDO-API-VERSION": HIRUNDO_API_VERSION,
+    }
+
+
+def get_headers():
+    return {
+        **_json_headers,
+        **_get_auth_headers(),
+        **_get_api_version_header(),
+    }

hirundo/_timeouts.py

@@ -1,2 +1,3 @@
 READ_TIMEOUT = 30.0
 MODIFY_TIMEOUT = 60.0
+DOWNLOAD_READ_TIMEOUT = 600.0  # 10 minutes

hirundo/_urls.py

@@ -0,0 +1,59 @@
+from typing import Annotated
+
+from pydantic import StringConstraints, UrlConstraints
+from pydantic_core import Url
+
+from hirundo.dataset_enum import StorageTypes
+
+S3BucketUrl = Annotated[
+    str,
+    StringConstraints(
+        min_length=8,
+        max_length=1023,
+        pattern=r"s3?://[a-z0-9.-]{3,64}[/]?",  # Only allow real S3 bucket URLs
+    ),
+]
+
+StorageConfigName = Annotated[
+    str,
+    StringConstraints(
+        min_length=1,
+        max_length=255,
+        pattern=r"^[a-zA-Z0-9-_]+$",
+    ),
+]
+
+STORAGE_PATTERNS: dict[StorageTypes, str] = {
+    StorageTypes.S3: r"^s3:\/\/[a-z0-9\.\-]{3,63}/[a-zA-Z0-9!\-\/_\.\*'\(\)]+$",
+    StorageTypes.GCP: r"^gs:\/\/([a-z0-9][a-z0-9_-]{1,61}[a-z0-9](\.[a-z0-9][a-z0-9_-]{1,61}[a-z0-9])*)\/[^\x00-\x1F\x7F-\x9F\r\n]*$",
+}
+
+
+LENGTH_CONSTRAINTS: dict[StorageTypes, dict] = {
+    StorageTypes.S3: {"min_length": 8, "max_length": 1023, "bucket_max_length": None},
+    StorageTypes.GCP: {"min_length": 8, "max_length": 1023, "bucket_max_length": 222},
+}
+
+RepoUrl = Annotated[
+    Url,
+    UrlConstraints(
+        allowed_schemes=[
+            "ssh",
+            "https",
+            "http",
+        ]
+    ),
+]
+HirundoUrl = Annotated[
+    Url,
+    UrlConstraints(
+        allowed_schemes=[
+            "file",
+            "https",
+            "http",
+            "s3",
+            "gs",
+            "ssh",
+        ]
+    ),
+]

hirundo/cli.py

@@ -7,6 +7,8 @@ from typing import Annotated
 from urllib.parse import urlparse
 
 import typer
+from rich.console import Console
+from rich.table import Table
 
 from hirundo._env import API_HOST, EnvLocation
 
@@ -189,6 +191,56 @@ def setup(
         )
 
 
+@app.command("check-run", epilog=hirundo_epilog)
+def check_run(
+    run_id: str,
+):
+    """
+    Check the status of a run.
+    """
+    from hirundo.dataset_optimization import OptimizationDataset
+
+    results = OptimizationDataset.check_run_by_id(run_id)
+    print(f"Run results saved to {results.cached_zip_path}")
+
+
+@app.command("list-runs", epilog=hirundo_epilog)
+def list_runs():
+    """
+    List all runs available.
+    """
+    from hirundo.dataset_optimization import OptimizationDataset
+
+    runs = OptimizationDataset.list_runs()
+
+    console = Console()
+    table = Table(
+        title="Runs:",
+        expand=True,
+    )
+    cols = (
+        "Dataset name",
+        "Run ID",
+        "Status",
+        "Created At",
+        "Run Args",
+    )
+    for col in cols:
+        table.add_column(
+            col,
+            overflow="fold",
+        )
+    for run in runs:
+        table.add_row(
+            str(run.name),
+            str(run.id),
+            str(run.status),
+            run.created_at.isoformat(),
+            run.run_args.model_dump_json() if run.run_args else None,
+        )
+    console.print(table)
+
+
 typer_click_object = typer.main.get_command(app)
 
 if __name__ == "__main__":

hirundo/dataset_enum.py

@@ -0,0 +1,46 @@
+from enum import Enum
+
+
+class LabelingType(str, Enum):
+    """
+    Enum indicate what type of labeling is used for the given dataset.
+    Supported types are:
+    """
+
+    SINGLE_LABEL_CLASSIFICATION = "SingleLabelClassification"
+    OBJECT_DETECTION = "ObjectDetection"
+    SPEECH_TO_TEXT = "SpeechToText"
+    OBJECT_SEGMENTATION = "ObjectSegmentation"
+    SEMANTIC_SEGMENTATION = "SemanticSegmentation"
+    PANOPTIC_SEGMENTATION = "PanopticSegmentation"
+
+
+class DatasetMetadataType(str, Enum):
+    """
+    Enum indicate what type of metadata is provided for the given dataset.
+    Supported types are:
+    """
+
+    HIRUNDO_CSV = "HirundoCSV"
+    COCO = "COCO"
+    YOLO = "YOLO"
+    KeylabsObjDetImages = "KeylabsObjDetImages"
+    KeylabsObjDetVideo = "KeylabsObjDetVideo"
+    KeylabsObjSegImages = "KeylabsObjSegImages"
+    KeylabsObjSegVideo = "KeylabsObjSegVideo"
+
+
+class StorageTypes(str, Enum):
+    """
+    Enum for the different types of storage configs.
+    Supported types are:
+    """
+
+    S3 = "S3"
+    GCP = "GCP"
+    # AZURE = "Azure"  TODO: Azure storage config is coming soon
+    GIT = "Git"
+    LOCAL = "Local"
+    """
+    Local storage config is only supported for on-premises installations.
+    """