oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/common/work_request.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+import sys
+import time
+from typing import Callable
+
+import oci
+from oci import Signer
+from tqdm.auto import tqdm
+from ads.common.oci_datascience import OCIDataScienceMixin
+
+logger = logging.getLogger(__name__)
+
+WORK_REQUEST_STOP_STATE = ("SUCCEEDED", "FAILED", "CANCELED")
+DEFAULT_WAIT_TIME = 1200
+DEFAULT_POLL_INTERVAL = 10
+WORK_REQUEST_PERCENTAGE = 100
+# default tqdm progress bar format: 
+# {l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' '{rate_fmt}{postfix}]
+# customize the bar format to remove the {n_fmt}/{total_fmt} from the right side
+DEFAULT_BAR_FORMAT = '{l_bar}{bar}| [{elapsed}<{remaining}, ' '{rate_fmt}{postfix}]'
+
+
+class DataScienceWorkRequest(OCIDataScienceMixin):
+    """Class for monitoring OCI WorkRequest and representing on tqdm progress bar. This class inherits
+    `OCIDataScienceMixin` so as to call its `client` attribute to interact with OCI backend.
+    """
+
+    def __init__(
+        self, 
+        id: str, 
+        description: str = "Processing",
+        config: dict = None, 
+        signer: Signer = None, 
+        client_kwargs: dict = None, 
+        **kwargs
+    ) -> None:
+        """Initializes ADSWorkRequest object.
+
+        Parameters
+        ----------
+        id: str
+            Work Request OCID.
+        description: str
+            Progress bar initial step description (Defaults to `Processing`).
+        config : dict, optional
+            OCI API key config dictionary to initialize 
+            oci.data_science.DataScienceClient (Defaults to None).
+        signer : oci.signer.Signer, optional
+            OCI authentication signer to initialize 
+            oci.data_science.DataScienceClient (Defaults to None).
+        client_kwargs : dict, optional
+            Additional client keyword arguments to initialize 
+            oci.data_science.DataScienceClient (Defaults to None).
+        kwargs:
+            Additional keyword arguments to initialize 
+            oci.data_science.DataScienceClient.
+        """
+        self.id = id
+        self._description = description
+        self._percentage = 0
+        self._status = None
+        super().__init__(config, signer, client_kwargs, **kwargs)
+        
+
+    def _sync(self):
+        """Fetches the latest work request information to ADSWorkRequest object."""
+        work_request = self.client.get_work_request(self.id).data
+        work_request_logs = self.client.list_work_request_logs(
+            self.id
+        ).data
+
+        self._percentage= work_request.percent_complete
+        self._status = work_request.status
+        self._description = work_request_logs[-1].message if work_request_logs else "Processing"
+
+    def watch(
+        self, 
+        progress_callback: Callable,
+        max_wait_time: int=DEFAULT_WAIT_TIME,
+        poll_interval: int=DEFAULT_POLL_INTERVAL,
+    ):
+        """Updates the progress bar with realtime message and percentage until the process is completed.
+
+        Parameters
+        ----------
+        progress_callback: Callable
+            Progress bar callback function.
+            It must accept `(percent_change, description)` where `percent_change` is the
+            work request percent complete and `description` is the latest work request log message. 
+        max_wait_time: int
+            Maximum amount of time to wait in seconds (Defaults to 1200).
+            Negative implies infinite wait time. 
+        poll_interval: int
+            Poll interval in seconds (Defaults to 10).
+
+        Returns
+        -------
+        None
+        """
+        previous_percent_complete = 0
+
+        start_time = time.time()
+        while self._percentage < 100:
+
+            seconds_since = time.time() - start_time
+            if max_wait_time > 0 and seconds_since >= max_wait_time:
+                logger.error(f"Exceeded max wait time of {max_wait_time} seconds.")
+                return
+
+            time.sleep(poll_interval)
+
+            try:
+                self._sync()
+            except Exception as ex:
+                logger.warn(ex)
+                continue
+
+            percent_change = self._percentage - previous_percent_complete
+            previous_percent_complete = self._percentage
+            progress_callback(
+                percent_change=percent_change,
+                description=self._description
+            )
+
+            if self._status in WORK_REQUEST_STOP_STATE:
+                if self._status != oci.work_requests.models.WorkRequest.STATUS_SUCCEEDED:
+                    if self._description:
+                        raise Exception(self._description)
+                    else:
+                        raise Exception(
+                            "Error occurred in attempt to perform the operation. "
+                            "Check the service logs to get more details. "
+                            f"Work request id: {self.id}."
+                        )
+                else:
+                    break
+
+        progress_callback(percent_change=0, description="Done")
+
+    def wait_work_request(
+        self,
+        progress_bar_description: str="Processing",
+        max_wait_time: int=DEFAULT_WAIT_TIME,
+        poll_interval: int=DEFAULT_POLL_INTERVAL
+    ):
+        """Waits for the work request progress bar to be completed.
+        
+        Parameters
+        ----------
+        progress_bar_description: str
+            Progress bar initial step description (Defaults to `Processing`).
+        max_wait_time: int
+            Maximum amount of time to wait in seconds (Defaults to 1200).
+            Negative implies infinite wait time.
+        poll_interval: int
+            Poll interval in seconds (Defaults to 10).
+        
+        Returns
+        -------
+        None
+        """
+
+        with tqdm(
+            total=WORK_REQUEST_PERCENTAGE,
+            leave=False,
+            mininterval=0,
+            file=sys.stdout,
+            desc=progress_bar_description,
+            bar_format=DEFAULT_BAR_FORMAT
+        ) as pbar:
+
+            def progress_callback(percent_change, description):
+                if percent_change != 0:
+                    pbar.update(percent_change)
+                if description:
+                    pbar.set_description(description)
+
+            self.watch(
+                progress_callback=progress_callback,
+                max_wait_time=max_wait_time,
+                poll_interval=poll_interval
+            )
+ 

ads/data_labeling/__init__.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.data_labeling.data_labeling_service import DataLabeling
+from ads.data_labeling.loader.file_loader import ImageFileLoader, TextFileLoader
+from ads.data_labeling.metadata import Metadata
+from ads.data_labeling.reader.dataset_reader import LabeledDatasetReader
+from ads.data_labeling.reader.metadata_reader import MetadataReader
+from ads.data_labeling.reader.record_reader import RecordReader
+from ads.data_labeling.constants import DatasetType, AnnotationType

ads/data_labeling/boundingbox.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from dataclasses import dataclass, field
+from typing import List, Tuple
+
+
+@dataclass
+class BoundingBoxItem:
+    """BoundingBoxItem class representing bounding box label.
+
+    Attributes
+    ----------
+    labels: List[str]
+        List of labels for this bounding box.
+    top_left: Tuple[float, float]
+        Top left corner of this bounding box.
+    bottom_left: Tuple[float, float]
+        Bottom left corner of this bounding box.
+    bottom_right: Tuple[float, float]
+        Bottom right corner of this bounding box.
+    top_right: Tuple[float, float]
+        Top right corner of this bounding box.
+
+    Examples
+    --------
+    >>> item = BoundingBoxItem(
+    ...     labels = ['cat','dog']
+    ...     bottom_left=(0.2, 0.4),
+    ...     top_left=(0.2, 0.2),
+    ...     top_right=(0.8, 0.2),
+    ...     bottom_right=(0.8, 0.4))
+    >>> item.to_yolo(categories = ['cat','dog', 'horse'])
+    """
+
+    top_left: Tuple[float, float]
+    bottom_left: Tuple[float, float]
+    bottom_right: Tuple[float, float]
+    top_right: Tuple[float, float]
+    labels: List[str] = field(default_factory=list)
+
+    def _validate(self):
+        """Validates the instance.
+
+        Raises
+        ------
+        ValueError
+            If the bounding box coordinate is not between [0.0, 1.0].
+        """
+        if (
+            not self.labels
+            or not isinstance(self.labels, list)
+            or len(self.labels) == 0
+        ):
+            raise ValueError(
+                "The parameter `df` is invalid. The BoundingBoxItem objects in the column "
+                f"{self.labels}  must contain a non-empty list of string labels."
+            )
+
+        if any(
+            (
+                not isinstance(entity, Tuple)
+                or len(entity) != 2
+                or not isinstance(entity[0], float)
+                or not isinstance(entity[1], float)
+                or not 0.0 <= entity[0] <= 1.0
+                or not 0.0 <= entity[1] <= 1.0
+            )
+            for entity in [
+                self.bottom_left,
+                self.top_left,
+                self.top_right,
+                self.bottom_right,
+            ]
+        ):
+            raise ValueError(
+                "The parameter `df` is invalid. The BoundingBoxItem objects in the column "
+                f"{[self.bottom_left, self.top_left, self.top_right, self.bottom_right,]} must "
+                "contain a tuple of two real numbers in the range of [0, 1]. "
+                "One BoundingBoxItem contains the values (invalid_tuple). Use the `DataLabeling.export()` "
+                "method to create a new dataset record file."
+            )
+
+    def __post_init__(self):
+        self._validate()
+
+    def to_yolo(
+        self, categories: List[str]
+    ) -> List[Tuple[int, float, float, float, float]]:
+        """Converts BoundingBoxItem to the YOLO format.
+
+        Parameters
+        ----------
+        categories: List[str]
+            The list of object categories in proper order for model training.
+            Example: ['cat','dog','horse']
+
+        Returns
+        -------
+        List[Tuple[int, float, float, float, float]]
+            The list of YOLO formatted bounding boxes.
+
+        Raises
+        ------
+        ValueError
+            When categories list not provided.
+            When categories list not matched with the labels.
+        TypeError
+            When categories list has a wrong format.
+        """
+        if not categories:
+            raise ValueError(
+                "The parameter `categories` is required. Use the `.info()` method to obtain a list of categories for this dataset."
+            )
+        if not isinstance(categories, list):
+            raise TypeError(
+                "The parameter `categories` is invalid. It must be an object of type `List[str]`."
+            )
+        if not set(self.labels).issubset(categories):
+            raise ValueError(
+                "The parameter `categories` is invalid. It must be a list of all the unique labels in the dataset."
+            )
+
+        category_map = {k: v for v, k in enumerate(categories)}
+        coords = (
+            (self.top_left[0] + self.top_right[0]) / 2,
+            (self.top_left[1] + self.bottom_left[1]) / 2,
+            self.top_right[0] - self.top_left[0],
+            self.bottom_left[1] - self.top_left[1],
+        )
+
+        return [(category_map[label],) + coords for label in self.labels]
+
+    @classmethod
+    def from_yolo(
+        cls, bbox: List[Tuple], categories: List[str] = None
+    ) -> "BoundingBoxItem":
+        """Converts the YOLO formated annotations to BoundingBoxItem.
+
+        Parameters
+        ----------
+        bboxes: List[Tuple]
+            The list of bounding box annotations in YOLO format.
+            Example: [(0, 0.511560675, 0.50234826, 0.47013485, 0.57803468)]
+        categories: List[str]
+            The list of object categories in proper order for model training.
+            Example: ['cat','dog','horse']
+
+        Returns
+        -------
+        BoundingBoxItem
+            The BoundingBoxItem.
+
+        Raises
+        ------
+        TypeError
+            When categories list has a wrong format.
+        """
+
+        if bbox:
+            _, x1, y1, w_size, h_size = bbox[0]
+            labels = [label[0] for label in bbox]
+            if categories:
+                if not isinstance(categories, list):
+                    raise TypeError("The categories must be a List[str].")
+                if max(labels) + 1 > len(categories):
+                    raise ValueError(
+                        "Index out of the range of the categories list. "
+                        "The categories must contain all the labels "
+                        "in the order that the integer label corresponds to."
+                    )
+
+                labels = [categories[label[0]] for label in bbox]
+
+            top_left = (x1 - (w_size / 2), y1 - (h_size / 2))
+            top_right = (x1 + (w_size / 2), y1 - (h_size / 2))
+            bottom_right = (x1 + (w_size / 2), y1 + (h_size / 2))
+            bottom_left = (x1 - (w_size / 2), y1 + (h_size / 2))
+            bbox_item = BoundingBoxItem(
+                top_left, bottom_left, bottom_right, top_right, labels
+            )
+            bbox_item._validate()
+            return bbox_item
+        else:
+            return None
+
+
+@dataclass
+class BoundingBoxItems:
+    """BoundingBoxItems class which consists of a list of BoundingBoxItem.
+
+    Attributes
+    ----------
+    items: List[BoundingBoxItem]
+        List of BoundingBoxItem.
+
+    Examples
+    --------
+    >>> item = BoundingBoxItem(
+    ...     labels = ['cat','dog']
+    ...     bottom_left=(0.2, 0.4),
+    ...     top_left=(0.2, 0.2),
+    ...     top_right=(0.8, 0.2),
+    ...     bottom_right=(0.8, 0.4))
+    >>> items = BoundingBoxItems(items = [item])
+    >>> items.to_yolo(categories = ['cat','dog', 'horse'])
+    """
+
+    items: List[BoundingBoxItem] = field(default_factory=list)
+
+    def __getitem__(self, index: int) -> BoundingBoxItem:
+        return self.items[index]
+
+    def to_yolo(
+        self, categories: List[str]
+    ) -> List[Tuple[int, float, float, float, float]]:
+        """Converts BoundingBoxItems to the YOLO format.
+
+        Parameters
+        ----------
+        categories: List[str]
+            The list of object categories in proper order for model training.
+            Example: ['cat','dog','horse']
+
+        Returns
+        -------
+        List[Tuple[int, float, float, float, float]]
+            The list of YOLO formatted bounding boxes.
+
+        Raises
+        ------
+        ValueError
+            When categories list not provided.
+            When categories list not matched with the labels.
+        TypeError
+            When categories list has a wrong format.
+        """
+        if not categories:
+            raise ValueError(
+                "The parameter `categories` is required. Use the `.info()` method to obtain a list of categories for this dataset."
+            )
+        if not isinstance(categories, list):
+            raise TypeError(
+                "The parameter `categories` is invalid. It must be an object of type `List[str]`."
+            )
+
+        result = []
+        for item in self.items:
+            result.extend(item.to_yolo(categories))
+        return result

ads/data_labeling/constants.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+class DatasetType:
+    """DatasetType class which contains all the dataset
+    types that data labeling service supports.
+    """
+
+    TEXT = "TEXT"
+    IMAGE = "IMAGE"
+    DOCUMENT = "DOCUMENT"
+
+
+class AnnotationType:
+    """AnnotationType class which contains all the annotation
+    types that data labeling service supports.
+    """
+
+    SINGLE_LABEL = "SINGLE_LABEL"
+    MULTI_LABEL = "MULTI_LABEL"
+    ENTITY_EXTRACTION = "ENTITY_EXTRACTION"
+    BOUNDING_BOX = "BOUNDING_BOX"
+
+
+class Formats:
+    """Common formats class which contains all the common
+    formats that are supported to convert to.
+    """
+
+    SPACY = "spacy"
+    YOLO = "yolo"
+
+
+FORMATS_TO_ANNOTATION_TYPE = {
+    Formats.SPACY: AnnotationType.ENTITY_EXTRACTION,
+    Formats.YOLO: AnnotationType.BOUNDING_BOX,
+}
+
+
+ANNOTATION_TYPE_TO_FORMATS = {
+    AnnotationType.ENTITY_EXTRACTION: Formats.SPACY,
+    AnnotationType.BOUNDING_BOX: Formats.YOLO,
+}