oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/data_labeling/data_labeling_service.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+import re
+import time
+from urllib.parse import urlparse
+
+import pandas as pd
+from ads.common import auth, oci_client, utils
+from ads.common.oci_mixin import OCIWorkRequestMixin
+from ads.common.utils import snake_to_camel
+from ads.config import JOB_RUN_COMPARTMENT_OCID, NB_SESSION_COMPARTMENT_OCID
+from ads.common.object_storage_details import (
+    ObjectStorageDetails,
+    InvalidObjectStoragePath,
+)
+from oci import pagination
+from oci.data_labeling_service.models import (
+    ObjectStorageSnapshotExportDetails,
+    SnapshotDatasetDetails,
+    WorkRequest,
+)
+from oci.util import to_dict
+
+logger = logging.getLogger(__name__)
+NUM_PROGRESS_BAR = 8
+
+
+class DataLabeling(OCIWorkRequestMixin):
+    """Class for data labeling service. Integrate the data labeling service APIs.
+
+    Examples
+    --------
+    >>> import ads
+    >>> import pandas
+    >>> from ads.data_labeling.data_labeling_service import DataLabeling
+    >>> ads.set_auth("api_key")
+    >>> dls = DataLabeling()
+    >>> dls.list_dataset()
+    >>> metadata_path = dls.export(dataset_id="your dataset id",
+    ...     path="oci://<bucket_name>@<namespace>/folder")
+    >>> df = pd.DataFrame.ads.read_labeled_data(metadata_path)
+    """
+
+    def __init__(
+        self,
+        compartment_id: str = None,
+        dls_cp_client_auth: dict = None,
+        dls_dp_client_auth: dict = None,
+    ) -> None:
+        """Initialize a DataLabeling class.
+
+        Parameters
+        ----------
+        compartment_id : str, optional
+            OCID of data labeling datasets' compartment
+        dls_cp_client_auth : dict, optional
+            Data Labeling control plane client auth. Default is None. The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+        dls_dp_client_auth : dict, optional
+            Data Labeling data plane client auth. Default is None. The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+
+        Returns
+        -------
+        None
+            Nothing.
+        """
+        self.compartment_id = compartment_id
+        if self.compartment_id is None:
+            self.compartment_id = (
+                NB_SESSION_COMPARTMENT_OCID or JOB_RUN_COMPARTMENT_OCID
+            )
+
+        if not self.compartment_id:
+            raise ValueError("The parameter `compartment_id` is required.")
+
+        self.dls_cp_client_auth = dls_cp_client_auth or auth.default_signer()
+        self.dls_dp_client_auth = dls_dp_client_auth or auth.default_signer()
+
+        self.dls_dp_client = oci_client.OCIClientFactory(
+            **self.dls_dp_client_auth
+        ).data_labeling_dp
+
+        self.dls_cp_client = oci_client.OCIClientFactory(
+            **self.dls_cp_client_auth
+        ).data_labeling_cp
+
+    def list_dataset(self, **kwargs) -> pd.DataFrame:
+        """List all the datasets created from the data labeling service under a given compartment.
+
+        Parameters
+        ----------
+        kwargs : dict, optional
+            Additional keyword arguments will be passed to oci.data_labeling_serviceDataLabelingManagementClient.list_datasets method.
+
+        Returns
+        -------
+        pandas.DataFrame
+            pandas dataframe which contains the dataset information.
+
+        Raises
+        ------
+        Exception
+            If pagination.list_call_get_all_results() fails
+        """
+        try:
+            items = pagination.list_call_get_all_results(
+                self.dls_cp_client.list_datasets, self.compartment_id, **kwargs
+            ).data
+        except Exception as e:
+            raise e
+
+        df = pd.DataFrame()
+        if items:
+            df = pd.concat(
+                [
+                    pd.DataFrame(to_dict(dataset), index=[i])
+                    for i, dataset in enumerate(items)
+                ],
+                axis=0,
+            )
+            df = (
+                df.reset_index(drop=True).set_index("id").drop(columns="compartment_id")
+            )
+        df.columns = [
+            snake_to_camel(name, capitalized_first_token=True) for name in df.columns
+        ]
+        return df
+
+    def export(self, dataset_id: str, path: str, include_unlabeled=False) -> str:
+        """Export dataset based on the dataset_id and save the jsonl files under the path
+        (metadata jsonl file and the records jsonl file) to the object storage path provided by the user
+        and return the metadata jsonl path.
+
+        Parameters
+        ----------
+        dataset_id : str
+            The dataset id of which the snapshot will be generated.
+        path : str
+            The object storage path to store the generated snapshot.
+            "oci://<bucket_name>@<namespace>/prefix"
+        include_unlabeled: bool, Optional. Defaults to False.
+            Whether to include unlabeled records or not.
+
+        Returns
+        -------
+        str
+            oci path of the metadata jsonl file.
+        """
+
+        if not re.match(r"oci://*@*", path):
+            raise InvalidObjectStoragePath(
+                "The parameter `path` is not valid. It must follow the pattern 'oci://<bucket_name>@<namespace>/key'."
+            )
+        url_parse = urlparse(path)
+        bucketname = url_parse.username
+        namespace = url_parse.hostname
+        if not bucketname:
+            raise InvalidObjectStoragePath(
+                f"The parameter `path` is not valid. The bucket name ({bucketname}) was not found. It must follow the pattern 'oci://<bucket_name>@<namespace>/key'."
+            )
+        if not namespace:
+            raise InvalidObjectStoragePath(
+                f"The parameter `path` is not valid. The name space ({namespace}) was not found. It must follow the pattern 'oci://<bucket_name>@<namespace>/key'."
+            )
+
+        prefix = url_parse.path.strip("/")
+        self.client = self.dls_cp_client
+
+        if not prefix.endswith("/"):
+            prefix = prefix + "/"
+        os_snapshot_export_detail = ObjectStorageSnapshotExportDetails(
+            export_type="OBJECT_STORAGE",
+            namespace=namespace,
+            bucket=bucketname,
+            prefix=prefix,
+        )
+
+        snapshot_detail = SnapshotDatasetDetails(
+            are_annotations_included=True,
+            are_unannotated_records_included=include_unlabeled,
+            export_details=os_snapshot_export_detail,
+        )
+        try:
+            snapshot_detail_response = self.dls_cp_client.snapshot_dataset(
+                dataset_id, snapshot_detail
+            )
+        except Exception as error:
+            if dataset_id not in self.list_dataset().index:
+                raise ValueError(
+                    "The parameter `dataset_id` is invalid. "
+                    "Use the `.list_dataset()` method to obtain a list of  all available datasets."
+                )
+            raise error
+
+        res_work_request = self._wait_for_work_request(snapshot_detail_response)
+
+        metadata = res_work_request.data.resources[1].metadata
+        return ObjectStorageDetails(
+            metadata["BUCKET"], metadata["NAMESPACE"], metadata["OBJECT"]
+        ).path
+
+    def _wait_for_work_request(self, snapshot_detail_response):
+        successful_state = WorkRequest.STATUS_SUCCEEDED
+        wait_for_states = (
+            WorkRequest.STATUS_CANCELED,
+            WorkRequest.STATUS_CANCELING,
+            WorkRequest.STATUS_FAILED,
+        )
+        work_request_id = snapshot_detail_response.headers["opc-work-request-id"]
+        work_request_log_entires = self.dls_cp_client.list_work_request_logs(
+            work_request_id
+        ).data.items
+        i = 0
+        res_work_request = self.dls_cp_client.get_work_request(work_request_id)
+        self._num_progress_bar = NUM_PROGRESS_BAR
+        with utils.get_progress_bar(self._num_progress_bar) as progress:
+            while len(work_request_log_entires) <= self._num_progress_bar - 1:
+                new_work_request_log_entires = work_request_log_entires[i:]
+                for work_request_log_entry in new_work_request_log_entires:
+                    progress.update(work_request_log_entry.message)
+                    i += 1
+                if (
+                    len(work_request_log_entires) == self._num_progress_bar - 1
+                    or res_work_request.data.status in wait_for_states
+                ):
+                    progress.update(work_request_log_entires[-1].message)
+                    if res_work_request.data.status != successful_state:
+                        raise Exception(work_request_log_entires[-1].message)
+                    else:
+                        break
+                time.sleep(i)
+                work_request_log_entires = self.dls_cp_client.list_work_request_logs(
+                    work_request_id
+                ).data.items
+                res_work_request = self.dls_cp_client.get_work_request(work_request_id)
+        return res_work_request

ads/data_labeling/interface/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/data_labeling/interface/loader.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class Loader(ABC):
+    """Data Loader Interface."""
+
+    @abstractmethod
+    def load(self, **kwargs) -> Any:
+        pass

ads/data_labeling/interface/parser.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class Parser(ABC):
+    """Data Parser Interface."""
+
+    @abstractmethod
+    def parse(self) -> Any:
+        pass

ads/data_labeling/interface/reader.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from abc import ABC, abstractmethod
+from typing import Any
+from ads.common.serializer import Serializable
+
+
+class Reader(ABC):
+    """Data Reader Interface."""
+
+    def info(self) -> Serializable:
+        NotImplementedError(
+            f"The class {self.__class__.__name__} did not implement the required method "
+            "`info()`. Contact the class maintainer."
+        )
+
+    @abstractmethod
+    def read(self) -> Any:
+        pass

ads/data_labeling/loader/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/data_labeling/loader/file_loader.py

@@ -0,0 +1,241 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from io import BytesIO
+from typing import Any, Dict, List, Union
+
+import fsspec
+import PIL
+from ads.common import auth as authutil
+from ads.data_labeling.constants import DatasetType
+from ads.data_labeling.interface.loader import Loader
+from ads.text_dataset.dataset import TextDatasetFactory, backends
+from PIL import Image
+
+THREAD_POOL_MAX_WORKERS = 10
+
+
+class FileLoader:
+    """FileLoader Base Class.
+
+    Attributes:
+    ----------
+    auth: (dict, optional). Defaults to None.
+        The default authetication is set using `ads.set_auth` API. If you need to override the
+        default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+        authentication signer and kwargs required to instantiate IdentityClient object.
+
+    Examples
+    --------
+    >>> from ads.data_labeling.loader.file_loader import FileLoader
+    >>> import oci
+    >>> import os
+    >>> from ads.common import auth as authutil
+    >>> path = "path/to/your_text_file.txt"
+    >>> file_content = FileLoader(auth=authutil.api_keys()).load(path)
+    """
+
+    def __init__(self, auth: Dict = None) -> "FileLoader":
+        """Initiates a FileLoader instance.
+
+        Parameters
+        ----------
+        auth: (dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+        """
+        self.auth = auth or authutil.default_signer()
+
+    def load(self, path: str, **kwargs) -> BytesIO:
+        """Loads the file content from the path.
+
+        Parameters
+        ----------
+        path: str
+            The file path, can be local or object storage path.
+        kwargs:
+            Nothing.
+
+        Returns
+        -------
+        BytesIO
+            The data in BytesIO format.
+        """
+        data = None
+        try:
+            with fsspec.open(path, **self.auth) as f:
+                data = f.read()
+        except FileNotFoundError:
+            raise FileNotFoundError(f"{path} not found.")
+        except Exception as e:
+            raise e
+
+        return data
+
+    def bulk_load(self, paths: List[str], **kwargs) -> Dict[str, Any]:
+        """Loads the files content from the list of paths.
+        The ThreadPoolExecutor is used to load the files in parallel threads.
+
+        Parameters
+        ----------
+        paths: List[str]
+            The list of file paths, can be local or object storage paths.
+
+        Returns
+        -------
+        Dict[str, Any]
+            The map between file path and file content.
+        """
+        result = {}
+        if not paths or not isinstance(paths, list) or len(paths) == 0:
+            return result
+
+        with ThreadPoolExecutor(max_workers=THREAD_POOL_MAX_WORKERS) as pool:
+            futures = {pool.submit(self.load, path, **kwargs): path for path in paths}
+            result = {futures[task]: task.result() for task in as_completed(futures)}
+        return result
+
+
+class TextFileLoader(FileLoader):
+    """
+    TextFileLoader class which loads text files.
+
+    Examples
+    --------
+    >>> from ads.data_labeling import TextFileLoader
+    >>> import oci
+    >>> import os
+    >>> from ads.common import auth as authutil
+    >>> path = "path/to/your_text_file.txt"
+    >>> file_content = TextFileLoader(auth=authutil.api_keys()).load(path)
+    """
+
+    def load(
+        self, path: str, backend: Union[str, backends.Base] = "default", **kwargs
+    ) -> str:
+        """Loads the content from the path.
+
+        Parameters
+        ----------
+        path: str
+            Text file path, can be local or object storage path.
+        backend: Union[str, backends.Base]
+            Default to "default". Valid options are "default" and "tika" or
+            ads.text_dataset.backends.Base, ads.text_dataset.backends.Tika
+        kwargs:
+            encoding: (str, optional). Defaults to 'utf-8'.
+                Encoding for text files. Used only to extract the content of the text dataset contents.
+
+        Returns
+        -------
+        str
+            Content of the text file.
+        """
+        format = os.path.splitext(path)[1].replace(".", "")
+        try:
+            content = next(
+                TextDatasetFactory.format(format.lower())
+                .backend(backend)
+                .read_text(path, storage_options=self.auth, **kwargs)
+            )[0]
+        except FileNotFoundError:
+            raise FileNotFoundError(f"{path} not found.")
+        except Exception as e:
+            raise e
+        return content
+
+
+class ImageFileLoader(FileLoader):
+    """
+    ImageFileLoader class which loads image files.
+
+    Examples
+    --------
+    >>> from ads.data_labeling import ImageFileLoader
+    >>> import oci
+    >>> import os
+    >>> from ads.common import auth as authutil
+    >>> path = "path/to/image.png"
+    >>> image = ImageFileLoader(auth=authutil.api_keys()).load(path)
+    """
+
+    def load(self, path: str, **kwargs) -> PIL.ImageFile.ImageFile:
+        """Loads the image from the path.
+
+        Parameters
+        ----------
+        path: str
+            Image file path, can be local or object storage path.
+        kwargs:
+            Nothing.
+
+        Returns
+        -------
+        PIL.ImageFile.ImageFile
+            Image opened by Pillow.
+        """
+        data = None
+        data = super().load(path=path)
+
+        return Image.open(BytesIO(data))
+
+
+class FileLoaderFactory:
+    """FileLoaderFactory class to create/register FileLoaders."""
+
+    _loaders = {
+        DatasetType.TEXT: TextFileLoader,
+        DatasetType.IMAGE: ImageFileLoader,
+        DatasetType.DOCUMENT: FileLoader,
+    }
+
+    @staticmethod
+    def loader(dataset_type: str, auth: Dict = None) -> FileLoader:
+        """Gets the loader based on the dataset_type.
+
+        Parameters
+        ----------
+        dataset_type: str
+            Dataset type. Currently supports TEXT, IMAGE and DOCUMENT.
+        auth: (dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+
+        Returns
+        -------
+        FileLoader
+            A FileLoader instance corresponding to the dataset_type.
+        """
+        if not dataset_type in FileLoaderFactory._loaders:
+            raise ValueError(
+                f"The wrong dataset type has been provided. "
+                f"Supported dataset types are: `{DatasetType.TEXT}`, "
+                f"`{DatasetType.IMAGE}` and `{DatasetType.DOCUMENT}`."
+            )
+
+        return FileLoaderFactory._loaders[dataset_type](auth=auth)
+
+    @classmethod
+    def register(cls, dataset_type: str, loader: Loader) -> None:
+        """Registers a new loader for a given dataset_type.
+
+        Parameters
+        ----------
+        dataset_type: str
+            Dataset type. Currently supports TEXT and IMAGE.
+        loader: Loader
+            A Loader class which supports loading content of the given dataset_type.
+
+        Returns
+        -------
+        None
+            Nothing.
+        """
+        cls._parsers[dataset_type] = loader

ads/data_labeling/metadata.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List
+
+import pandas as pd
+from ads.common.object_storage_details import ObjectStorageDetails
+from ads.common.serializer import DataClassSerializable
+from oci.data_labeling_service_dataplane.models.dataset import Dataset as OCIDLSDataset
+
+
+@dataclass
+class Metadata(DataClassSerializable):
+    """The class that representing the labeled dataset metadata.
+
+    Attributes
+    ----------
+    source_path: str
+        Contains information on where all the source data(image/text/document) stores.
+    records_path: str
+        Contains information on where records jsonl file stores.
+    labels: List
+        List of classes/labels for the dataset.
+    dataset_name: str
+        Dataset display name on the Data Labeling Service console.
+    compartment_id: str
+        Compartment id of the labeled dataset.
+    dataset_id: str
+        Dataset id.
+    annotation_type: str
+        Type of the labeling/annotation task. Currently supports SINGLE_LABEL,
+        MULTI_LABEL, ENTITY_EXTRACTION, BOUNDING_BOX.
+    dataset_type: str
+        Type of the dataset. Currently supports Text, Image, DOCUMENT.
+    """
+
+    source_path: str = ""
+    records_path: str = ""
+    labels: List[str] = field(default_factory=list)
+    dataset_name: str = ""
+    compartment_id: str = ""
+    dataset_id: str = ""
+    annotation_type: str = ""
+    dataset_type: str = ""
+
+    def to_dict(self) -> Dict:
+        """Converts to dictionary representation.
+
+        Returns
+        -------
+        Dict
+            The metadata in dictionary type.
+        """
+        return asdict(self)
+
+    def __repr__(self):
+        """Show the Metadata in yaml format."""
+        return self.to_yaml()
+
+    def to_dataframe(self) -> pd.DataFrame:
+        """
+        Converts the metadata to dataframe format.
+
+        Returns
+        -------
+        pandas.DataFrame
+            The metadata in Pandas dataframe format.
+        """
+        return pd.DataFrame({"": self.to_dict()})
+
+    def _repr_html_(self):
+        """Shows metadata in dataframe format."""
+        return (
+            self.to_dataframe().style.set_properties(**{"margin-left": "0px"}).to_html()
+        )
+
+    @classmethod
+    def from_dls_dataset(cls, dataset: OCIDLSDataset) -> "Metadata":
+        """Contructs a Metadata instance from OCI DLS dataset.
+
+        Parameters
+        ----------
+        dataset: OCIDLSDataset
+            OCIDLSDataset object.
+
+        Returns
+        -------
+        Metadata
+            The ads labeled dataset metadata instance.
+        """
+        oci_labels = [d.name for d in dataset.label_set.items]
+        oci_source_path = ObjectStorageDetails(
+            dataset.dataset_source_details.bucket,
+            dataset.dataset_source_details.namespace,
+            dataset.dataset_source_details.prefix,
+        ).path
+
+        return Metadata(
+            annotation_type=dataset.annotation_format,
+            dataset_name=dataset.display_name,
+            source_path=oci_source_path,
+            labels=oci_labels,
+            compartment_id=dataset.compartment_id,
+            dataset_id=dataset.id,
+            dataset_type=dataset.dataset_format_details.format_type,
+        )

ads/data_labeling/mixin/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/