hirundo 0.1.16__py3-none-any.whl → 0.1.21__py3-none-any.whl

hirundo/{dataset_optimization_results.py → dataset_qa_results.py}

@@ -21,7 +21,7 @@ if has_polars:
 T = typing.TypeVar("T")
 
 
-class DatasetOptimizationResults(BaseModel, typing.Generic[T]):
+class DatasetQAResults(BaseModel, typing.Generic[T]):
     model_config = {"arbitrary_types_allowed": True}
 
     cached_zip_path: Path
@@ -30,13 +30,13 @@ class DatasetOptimizationResults(BaseModel, typing.Generic[T]):
     """
     suspects: T
     """
-    A polars/pandas DataFrame containing the results of the optimization run
+    A polars/pandas DataFrame containing the results of the data QA run
     """
     object_suspects: typing.Optional[T]
     """
-    A polars/pandas DataFrame containing the object-level results of the optimization run
+    A polars/pandas DataFrame containing the object-level results of the data QA run
     """
     warnings_and_errors: T
     """
-    A polars/pandas DataFrame containing the warnings and errors of the optimization run
+    A polars/pandas DataFrame containing the warnings and errors of the data QA run
     """

hirundo/git.py

@@ -3,15 +3,14 @@ import re
 import typing
 
 import pydantic
-import requests
 from pydantic import BaseModel, field_validator
 from pydantic_core import Url
 
-from hirundo._constraints import RepoUrl
 from hirundo._env import API_HOST
 from hirundo._headers import get_headers
-from hirundo._http import raise_for_status_with_reason
+from hirundo._http import raise_for_status_with_reason, requests
 from hirundo._timeouts import MODIFY_TIMEOUT, READ_TIMEOUT
+from hirundo._urls import RepoUrl
 from hirundo.logger import get_logger
 
 logger = get_logger(__name__)

hirundo/labeling.py

@@ -0,0 +1,140 @@
+import typing
+from abc import ABC
+
+from pydantic import BaseModel, Field
+
+from hirundo.dataset_enum import DatasetMetadataType
+
+if typing.TYPE_CHECKING:
+    from hirundo._urls import HirundoUrl
+
+
+class Metadata(BaseModel, ABC, frozen=True):
+    type: DatasetMetadataType
+
+
+class HirundoCSV(Metadata, frozen=True):
+    """
+    A dataset metadata file in the Hirundo CSV format
+    """
+
+    type: typing.Literal[DatasetMetadataType.HIRUNDO_CSV] = (
+        DatasetMetadataType.HIRUNDO_CSV
+    )
+    csv_url: "HirundoUrl"
+    """
+    The URL to access the dataset metadata CSV file.
+    e.g. `s3://my-bucket-name/my-folder/my-metadata.csv`, `gs://my-bucket-name/my-folder/my-metadata.csv`,
+    or `ssh://my-username@my-repo-name/my-folder/my-metadata.csv`
+    (or `file:///datasets/my-folder/my-metadata.csv` if using LOCAL storage type with on-premises installation)
+    """
+
+
+class COCO(Metadata, frozen=True):
+    """
+    A dataset metadata file in the COCO format
+    """
+
+    type: typing.Literal[DatasetMetadataType.COCO] = DatasetMetadataType.COCO
+    json_url: "HirundoUrl"
+    """
+    The URL to access the dataset metadata JSON file.
+    e.g. `s3://my-bucket-name/my-folder/my-metadata.json`, `gs://my-bucket-name/my-folder/my-metadata.json`,
+    or `ssh://my-username@my-repo-name/my-folder/my-metadata.json`
+    (or `file:///datasets/my-folder/my-metadata.json` if using LOCAL storage type with on-premises installation)
+    """
+
+
+class YOLO(Metadata, frozen=True):
+    type: typing.Literal[DatasetMetadataType.YOLO] = DatasetMetadataType.YOLO
+    data_yaml_url: "typing.Optional[HirundoUrl]" = None
+    labels_dir_url: "HirundoUrl"
+
+
+class KeylabsAuth(BaseModel):
+    username: str
+    password: str
+    instance: str
+
+
+class Keylabs(Metadata, frozen=True):
+    project_id: str
+    """
+    Keylabs project ID.
+    """
+
+    labels_dir_url: "HirundoUrl"
+    """
+    URL to the directory containing the Keylabs labels.
+    """
+
+    with_attributes: bool = True
+    """
+    Whether to include attributes in the class name.
+    """
+
+    project_name: typing.Optional[str] = None
+    """
+    Keylabs project name (optional; added to output CSV if provided).
+    """
+    keylabs_auth: typing.Optional[KeylabsAuth] = None
+    """
+    Keylabs authentication credentials (optional; if provided, used to provide links to each sample).
+    """
+
+
+class KeylabsObjDetImages(Keylabs, frozen=True):
+    type: typing.Literal[DatasetMetadataType.KeylabsObjDetImages] = (
+        DatasetMetadataType.KeylabsObjDetImages
+    )
+
+
+class KeylabsObjDetVideo(Keylabs, frozen=True):
+    type: typing.Literal[DatasetMetadataType.KeylabsObjDetVideo] = (
+        DatasetMetadataType.KeylabsObjDetVideo
+    )
+
+
+class KeylabsObjSegImages(Keylabs, frozen=True):
+    type: typing.Literal[DatasetMetadataType.KeylabsObjSegImages] = (
+        DatasetMetadataType.KeylabsObjSegImages
+    )
+
+
+class KeylabsObjSegVideo(Keylabs, frozen=True):
+    type: typing.Literal[DatasetMetadataType.KeylabsObjSegVideo] = (
+        DatasetMetadataType.KeylabsObjSegVideo
+    )
+
+
+KeylabsInfo = typing.Union[
+    KeylabsObjDetImages, KeylabsObjDetVideo, KeylabsObjSegImages, KeylabsObjSegVideo
+]
+"""
+The dataset labeling info for Keylabs. The dataset labeling info can be one of the following:
+- `DatasetMetadataType.KeylabsObjDetImages`: Indicates that the dataset metadata file is in the Keylabs object detection image format
+- `DatasetMetadataType.KeylabsObjDetVideo`: Indicates that the dataset metadata file is in the Keylabs object detection video format
+- `DatasetMetadataType.KeylabsObjSegImages`: Indicates that the dataset metadata file is in the Keylabs object segmentation image format
+- `DatasetMetadataType.KeylabsObjSegVideo`: Indicates that the dataset metadata file is in the Keylabs object segmentation video format
+"""
+LabelingInfo = typing.Annotated[
+    typing.Union[
+        HirundoCSV,
+        COCO,
+        YOLO,
+        KeylabsInfo,
+    ],
+    Field(discriminator="type"),
+]
+"""
+The dataset labeling info. The dataset labeling info can be one of the following:
+- `DatasetMetadataType.HirundoCSV`: Indicates that the dataset metadata file is a CSV file with the Hirundo format
+- `DatasetMetadataType.COCO`: Indicates that the dataset metadata file is a JSON file with the COCO format
+- `DatasetMetadataType.YOLO`: Indicates that the dataset metadata file is in the YOLO format
+- `DatasetMetadataType.KeylabsObjDetImages`: Indicates that the dataset metadata file is in the Keylabs object detection image format
+- `DatasetMetadataType.KeylabsObjDetVideo`: Indicates that the dataset metadata file is in the Keylabs object detection video format
+- `DatasetMetadataType.KeylabsObjSegImages`: Indicates that the dataset metadata file is in the Keylabs object segmentation image format
+- `DatasetMetadataType.KeylabsObjSegVideo`: Indicates that the dataset metadata file is in the Keylabs object segmentation video format
+
+Currently no other formats are supported. Future versions of `hirundo` may support additional formats.
+"""

hirundo/storage.py

@@ -1,17 +1,16 @@
 import typing
-from enum import Enum
 from pathlib import Path
 
 import pydantic
-import requests
 from pydantic import BaseModel, model_validator
 from pydantic_core import Url
 
-from hirundo._constraints import S3BucketUrl, StorageConfigName
 from hirundo._env import API_HOST
 from hirundo._headers import get_headers
-from hirundo._http import raise_for_status_with_reason
+from hirundo._http import raise_for_status_with_reason, requests
 from hirundo._timeouts import MODIFY_TIMEOUT, READ_TIMEOUT
+from hirundo._urls import S3BucketUrl, StorageConfigName
+from hirundo.dataset_enum import StorageTypes
 from hirundo.git import GitRepo, GitRepoOut
 from hirundo.logger import get_logger
 
@@ -34,11 +33,11 @@ class StorageS3Base(BaseModel):
         Chains the bucket URL with the path, ensuring that the path is formatted correctly
 
         Args:
-            path: The path to the file in the S3 bucket, e.g. `my-file.txt` or `/my-folder/my-file.txt`
+            path: The path to the file in the S3 bucket, e.g. :file:`my-file.txt` or :file:`/my-folder/my-file.txt`
 
         Returns:
-            The full URL to the file in the S3 bucket, e.g. `s3://my-bucket/my-file.txt` or `s3://my-bucket/my-folder/my-file.txt`,
-            where `s3://my-bucket` is the bucket URL provided in the S3 storage config
+            The full URL to the file in the S3 bucket, e.g. :file:`s3://my-bucket/my-file.txt` or :file:`s3://my-bucket/my-folder/my-file.txt`,
+            where :file:`s3://my-bucket` is the bucket URL provided in the S3 storage config
         """
         return Url(
             f"{S3_PREFIX}{self.bucket_url.removeprefix(S3_PREFIX).removesuffix('/')}/{str(path).removeprefix('/')}"
@@ -64,11 +63,11 @@ class StorageGCPBase(BaseModel):
         Chains the bucket URL with the path, ensuring that the path is formatted correctly
 
         Args:
-            path: The path to the file in the GCP bucket, e.g. `my-file.txt` or `/my-folder/my-file.txt`
+            path: The path to the file in the GCP bucket, e.g. :file:`my-file.txt` or :file:`/my-folder/my-file.txt`
 
         Returns:
-            The full URL to the file in the GCP bucket, e.g. `gs://my-bucket/my-file.txt` or `gs://my-bucket/my-folder/my-file.txt`,
-            where `my-bucket` is the bucket name provided in the GCP storage config
+            The full URL to the file in the GCP bucket, e.g. :file:`gs://my-bucket/my-file.txt` or :file:`gs://my-bucket/my-folder/my-file.txt`,
+            where :file:`my-bucket` is the bucket name provided in the GCP storage config
         """
         return Url(f"gs://{self.bucket_name}/{str(path).removeprefix('/')}")
 
@@ -94,7 +93,7 @@ class StorageGCPOut(StorageGCPBase):
 #         Chains the container URL with the path, ensuring that the path is formatted correctly
 
 #         Args:
-#             path: The path to the file in the Azure container, e.g. `my-file.txt` or `/my-folder/my-file.txt`
+#             path: The path to the file in the Azure container, e.g. :file:`my-file.txt` or :file:`/my-folder/my-file.txt`
 
 #         Returns:
 #             The full URL to the file in the Azure container
@@ -114,11 +113,11 @@ def get_git_repo_url(
     Chains the repository URL with the path, ensuring that the path is formatted correctly
 
     Args:
-        repo_url: The URL of the git repository, e.g. `https://my-git-repository.com`
-        path: The path to the file in the git repository, e.g. `my-file.txt` or `/my-folder/my-file.txt`
+        repo_url: The URL of the git repository, e.g. :file:`https://my-git-repository.com`
+        path: The path to the file in the git repository, e.g. :file:`my-file.txt` or :file:`/my-folder/my-file.txt`
 
     Returns:
-        The full URL to the file in the git repository, e.g. `https://my-git-repository.com/my-file.txt` or `https://my-git-repository.com/my-folder/my-file.txt`
+        The full URL to the file in the git repository, e.g. :file:`https://my-git-repository.com/my-file.txt` or :file:`https://my-git-repository.com/my-folder/my-file.txt`
     """
     if not isinstance(repo_url, Url):
         repo_url = Url(repo_url)
@@ -131,12 +130,12 @@ class StorageGit(BaseModel):
     repo_id: typing.Optional[int] = None
     """
     The ID of the Git repository in the Hirundo system.
-    Either `repo_id` or `repo` must be provided.
+    Either :code:`repo_id` or :code:`repo` must be provided.
     """
     repo: typing.Optional[GitRepo] = None
     """
     The Git repository to link to.
-    Either `repo_id` or `repo` must be provided.
+    Either :code:`repo_id` or :code:`repo` must be provided.
     """
     branch: str
     """
@@ -156,11 +155,11 @@ class StorageGit(BaseModel):
         Chains the repository URL with the path, ensuring that the path is formatted correctly
 
         Args:
-            path: The path to the file in the git repository, e.g. `my-file.txt` or `/my-folder/my-file.txt`
+            path: The path to the file in the git repository, e.g. :file:`my-file.txt` or :file:`/my-folder/my-file.txt`
 
         Returns:
-            The full URL to the file in the git repository, e.g. `https://my-git-repository.com/my-file.txt` or `https://my-git-repository.com/my-folder/my-file.txt`,
-            where `https://my-git-repository.com` is the repository URL provided in the git storage config's git repo
+            The full URL to the file in the git repository, e.g. :file:`https://my-git-repository.com/my-file.txt` or :file:`https://my-git-repository.com/my-folder/my-file.txt`,
+            where :file:`https://my-git-repository.com` is the repository URL provided in the git storage config's git repo
         """
         if not self.repo:
             raise ValueError("Repo must be provided to use `get_url`")
@@ -179,47 +178,31 @@ class StorageGitOut(BaseModel):
         Chains the repository URL with the path, ensuring that the path is formatted correctly
 
         Args:
-            path: The path to the file in the git repository, e.g. `my-file.txt` or `/my-folder/my-file.txt`
+            path: The path to the file in the git repository, e.g. :file:`my-file.txt` or :file:`/my-folder/my-file.txt`
 
         Returns:
-            The full URL to the file in the git repository, e.g. `https://my-git-repository.com/my-file.txt` or `https://my-git-repository.com/my-folder/my-file.txt`,
-            where `https://my-git-repository.com` is the repository URL provided in the git storage config's git repo
+            The full URL to the file in the git repository, e.g. :file:`https://my-git-repository.com/my-file.txt` or :file:`https://my-git-repository.com/my-folder/my-file.txt`,
+            where :file:`https://my-git-repository.com` is the repository URL provided in the git storage config's git repo
         """
         repo_url = self.repo.repository_url
         return get_git_repo_url(repo_url, path)
 
 
-class StorageTypes(str, Enum):
-    """
-    Enum for the different types of storage configs.
-    Supported types are:
-    """
-
-    S3 = "S3"
-    GCP = "GCP"
-    # AZURE = "Azure"  TODO: Azure storage config is coming soon
-    GIT = "Git"
-    LOCAL = "Local"
-    """
-    Local storage config is only supported for on-premises installations.
-    """
-
-
 class StorageConfig(BaseModel):
     id: typing.Optional[int] = None
     """
-    The ID of the `StorageConfig` in the Hirundo system.
+    The ID of the :code:`StorageConfig` in the Hirundo system.
     """
 
     organization_id: typing.Optional[int] = None
     """
-    The ID of the organization that the `StorageConfig` belongs to.
+    The ID of the organization that the :code:`StorageConfig` belongs to.
     If not provided, it will be assigned to your default organization.
     """
 
     name: StorageConfigName
     """
-    A name to identify the `StorageConfig` in the Hirundo system.
+    A name to identify the :code:`StorageConfig` in the Hirundo system.
     """
     type: typing.Optional[StorageTypes] = pydantic.Field(
         examples=[
@@ -230,12 +213,12 @@ class StorageConfig(BaseModel):
         ]
     )
     """
-    The type of the `StorageConfig`.
+    The type of the :code:`StorageConfig`.
     Supported types are:
-    - `S3`
-    - `GCP`
-    - `Azure` (coming soon)
-    - `Git`
+    - :code:`S3`
+    - :code:`GCP`
+    - :code:`Azure` (coming soon)
+    - :code:`Git`
     """
     s3: typing.Optional[StorageS3] = pydantic.Field(
         default=None,
@@ -323,10 +306,10 @@ class StorageConfig(BaseModel):
     @staticmethod
     def get_by_id(storage_config_id: int) -> "ResponseStorageConfig":
         """
-        Retrieves a `StorageConfig` instance from the server by its ID
+        Retrieves a :code:`StorageConfig` instance from the server by its ID
 
         Args:
-            storage_config_id: The ID of the `StorageConfig` to retrieve
+            storage_config_id: The ID of the :code:`StorageConfig` to retrieve
         """
         storage_config = requests.get(
             f"{API_HOST}/storage-config/{storage_config_id}",
@@ -339,11 +322,11 @@ class StorageConfig(BaseModel):
     @staticmethod
     def get_by_name(name: str, storage_type: StorageTypes) -> "ResponseStorageConfig":
         """
-        Retrieves a `StorageConfig` instance from the server by its name
+        Retrieves a :code:`StorageConfig` instance from the server by its name
 
         Args:
-            name: The name of the `StorageConfig` to retrieve
-            storage_type: The type of the `StorageConfig` to retrieve
+            name: The name of the :code:`StorageConfig` to retrieve
+            storage_type: The type of the :code:`StorageConfig` to retrieve
 
             Note: The type is required because the name is not unique across different storage types
         """
@@ -360,12 +343,12 @@ class StorageConfig(BaseModel):
         organization_id: typing.Optional[int] = None,
     ) -> list["ResponseStorageConfig"]:
         """
-        Lists all the `StorageConfig`'s created by user's default organization
-        Note: The return type is `list[dict]` and not `list[StorageConfig]`
+        Lists all the :code:`StorageConfig`'s created by user's default organization
+        Note: The return type is :code:`list[dict]` and not :code:`list[StorageConfig]`
 
         Args:
-            organization_id: The ID of the organization to list `StorageConfig`'s for.
-            If not provided, it will list `StorageConfig`'s for the default organization.
+            organization_id: The ID of the organization to list :code:`StorageConfig`'s for.
+            If not provided, it will list :code:`StorageConfig`'s for the default organization.
         """
         storage_configs = requests.get(
             f"{API_HOST}/storage-config/",
@@ -379,10 +362,10 @@ class StorageConfig(BaseModel):
     @staticmethod
     def delete_by_id(storage_config_id) -> None:
         """
-        Deletes a `StorageConfig` instance from the server by its ID
+        Deletes a :code:`StorageConfig` instance from the server by its ID
 
         Args:
-            storage_config_id: The ID of the `StorageConfig` to delete
+            storage_config_id: The ID of the :code:`StorageConfig` to delete
         """
         storage_config = requests.delete(
             f"{API_HOST}/storage-config/{storage_config_id}",
@@ -394,7 +377,7 @@ class StorageConfig(BaseModel):
 
     def delete(self) -> None:
         """
-        Deletes the `StorageConfig` instance from the server
+        Deletes the :code:`StorageConfig` instance from the server
         """
         if not self.id:
             raise ValueError("No StorageConfig has been created")
@@ -402,10 +385,10 @@ class StorageConfig(BaseModel):
 
     def create(self, replace_if_exists: bool = False) -> int:
         """
-        Create a `StorageConfig` instance on the server
+        Create a :code:`StorageConfig` instance on the server
 
         Args:
-            replace_if_exists: If a `StorageConfig` with the same name and type already exists, replace it.
+            replace_if_exists: If a :code:`StorageConfig` with the same name and type already exists, replace it.
         """
         if self.git and self.git.repo:
             self.git.repo_id = self.git.repo.create(replace_if_exists=replace_if_exists)

hirundo/unzip.py

@@ -4,7 +4,6 @@ from collections.abc import Mapping
 from pathlib import Path
 from typing import IO, cast
 
-import requests
 from pydantic_core import Url
 
 from hirundo._dataframe import (
@@ -18,10 +17,11 @@ from hirundo._dataframe import (
 )
 from hirundo._env import API_HOST
 from hirundo._headers import _get_auth_headers
+from hirundo._http import requests
 from hirundo._timeouts import DOWNLOAD_READ_TIMEOUT
-from hirundo.dataset_optimization_results import (
+from hirundo.dataset_qa_results import (
     DataFrameType,
-    DatasetOptimizationResults,
+    DatasetQAResults,
 )
 from hirundo.logger import get_logger
 
@@ -117,7 +117,7 @@ def get_mislabel_suspect_filename(filenames: list[str]):
 
 def download_and_extract_zip(
     run_id: str, zip_url: str
-) -> DatasetOptimizationResults[DataFrameType]:
+) -> DatasetQAResults[DataFrameType]:
     """
     Download and extract the zip file from the given URL.
 
@@ -127,11 +127,11 @@ def download_and_extract_zip(
     and `warnings_and_errors.csv` files from the zip file.
 
     Args:
-        run_id: The ID of the optimization run.
+        run_id: The ID of the dataset QA run.
         zip_url: The URL of the zip file to download.
 
     Returns:
-        The dataset optimization results object.
+        The dataset QA results object.
     """
     # Define the local file path
     cache_dir = Path.home() / ".hirundo" / "cache"
@@ -140,9 +140,8 @@ def download_and_extract_zip(
 
     headers = None
     if Url(zip_url).scheme == "file":
-        zip_url = (
-            f"{API_HOST}/dataset-optimization/run/local-download"
-            + zip_url.replace("file://", "")
+        zip_url = f"{API_HOST}/dataset-qa/run/local-download" + zip_url.replace(
+            "file://", ""
         )
         headers = _get_auth_headers()
     # Stream the zip file download
@@ -217,7 +216,7 @@ def download_and_extract_zip(
                     "Failed to load warnings and errors into DataFrame", exc_info=e
                 )
 
-            return DatasetOptimizationResults[DataFrameType](
+            return DatasetQAResults[DataFrameType](
                 cached_zip_path=zip_file_path,
                 suspects=suspects_df,
                 object_suspects=object_suspects_df,