oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/data_labeling/data_labeling_service.py

@@ -1,244 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import logging
-import re
-import time
-from urllib.parse import urlparse
-
-import pandas as pd
-from ads.common import auth, oci_client, utils
-from ads.common.oci_mixin import OCIWorkRequestMixin
-from ads.common.utils import snake_to_camel
-from ads.config import JOB_RUN_COMPARTMENT_OCID, NB_SESSION_COMPARTMENT_OCID
-from ads.common.object_storage_details import (
-    ObjectStorageDetails,
-    InvalidObjectStoragePath,
-)
-from oci import pagination
-from oci.data_labeling_service.models import (
-    ObjectStorageSnapshotExportDetails,
-    SnapshotDatasetDetails,
-    WorkRequest,
-)
-from oci.util import to_dict
-
-logger = logging.getLogger(__name__)
-NUM_PROGRESS_BAR = 8
-
-
-class DataLabeling(OCIWorkRequestMixin):
-    """Class for data labeling service. Integrate the data labeling service APIs.
-
-    Examples
-    --------
-    >>> import ads
-    >>> import pandas
-    >>> from ads.data_labeling.data_labeling_service import DataLabeling
-    >>> ads.set_auth("api_key")
-    >>> dls = DataLabeling()
-    >>> dls.list_dataset()
-    >>> metadata_path = dls.export(dataset_id="your dataset id",
-    ...     path="oci://<bucket_name>@<namespace>/folder")
-    >>> df = pd.DataFrame.ads.read_labeled_data(metadata_path)
-    """
-
-    def __init__(
-        self,
-        compartment_id: str = None,
-        dls_cp_client_auth: dict = None,
-        dls_dp_client_auth: dict = None,
-    ) -> None:
-        """Initialize a DataLabeling class.
-
-        Parameters
-        ----------
-        compartment_id : str, optional
-            OCID of data labeling datasets' compartment
-        dls_cp_client_auth : dict, optional
-            Data Labeling control plane client auth. Default is None. The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        dls_dp_client_auth : dict, optional
-            Data Labeling data plane client auth. Default is None. The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-
-        Returns
-        -------
-        None
-            Nothing.
-        """
-        self.compartment_id = compartment_id
-        if self.compartment_id is None:
-            self.compartment_id = (
-                NB_SESSION_COMPARTMENT_OCID or JOB_RUN_COMPARTMENT_OCID
-            )
-
-        if not self.compartment_id:
-            raise ValueError("The parameter `compartment_id` is required.")
-
-        self.dls_cp_client_auth = dls_cp_client_auth or auth.default_signer()
-        self.dls_dp_client_auth = dls_dp_client_auth or auth.default_signer()
-
-        self.dls_dp_client = oci_client.OCIClientFactory(
-            **self.dls_dp_client_auth
-        ).data_labeling_dp
-
-        self.dls_cp_client = oci_client.OCIClientFactory(
-            **self.dls_cp_client_auth
-        ).data_labeling_cp
-
-    def list_dataset(self, **kwargs) -> pd.DataFrame:
-        """List all the datasets created from the data labeling service under a given compartment.
-
-        Parameters
-        ----------
-        kwargs : dict, optional
-            Additional keyword arguments will be passed to oci.data_labeling_serviceDataLabelingManagementClient.list_datasets method.
-
-        Returns
-        -------
-        pandas.DataFrame
-            pandas dataframe which contains the dataset information.
-
-        Raises
-        ------
-        Exception
-            If pagination.list_call_get_all_results() fails
-        """
-        try:
-            items = pagination.list_call_get_all_results(
-                self.dls_cp_client.list_datasets, self.compartment_id, **kwargs
-            ).data
-        except Exception as e:
-            raise e
-
-        df = pd.DataFrame()
-        if items:
-            df = pd.concat(
-                [
-                    pd.DataFrame(to_dict(dataset), index=[i])
-                    for i, dataset in enumerate(items)
-                ],
-                axis=0,
-            )
-            df = (
-                df.reset_index(drop=True).set_index("id").drop(columns="compartment_id")
-            )
-        df.columns = [
-            snake_to_camel(name, capitalized_first_token=True) for name in df.columns
-        ]
-        return df
-
-    def export(self, dataset_id: str, path: str, include_unlabeled=False) -> str:
-        """Export dataset based on the dataset_id and save the jsonl files under the path
-        (metadata jsonl file and the records jsonl file) to the object storage path provided by the user
-        and return the metadata jsonl path.
-
-        Parameters
-        ----------
-        dataset_id : str
-            The dataset id of which the snapshot will be generated.
-        path : str
-            The object storage path to store the generated snapshot.
-            "oci://<bucket_name>@<namespace>/prefix"
-        include_unlabeled: bool, Optional. Defaults to False.
-            Whether to include unlabeled records or not.
-
-        Returns
-        -------
-        str
-            oci path of the metadata jsonl file.
-        """
-
-        if not re.match(r"oci://*@*", path):
-            raise InvalidObjectStoragePath(
-                "The parameter `path` is not valid. It must follow the pattern 'oci://<bucket_name>@<namespace>/key'."
-            )
-        url_parse = urlparse(path)
-        bucketname = url_parse.username
-        namespace = url_parse.hostname
-        if not bucketname:
-            raise InvalidObjectStoragePath(
-                f"The parameter `path` is not valid. The bucket name ({bucketname}) was not found. It must follow the pattern 'oci://<bucket_name>@<namespace>/key'."
-            )
-        if not namespace:
-            raise InvalidObjectStoragePath(
-                f"The parameter `path` is not valid. The name space ({namespace}) was not found. It must follow the pattern 'oci://<bucket_name>@<namespace>/key'."
-            )
-
-        prefix = url_parse.path.strip("/")
-        self.client = self.dls_cp_client
-
-        if not prefix.endswith("/"):
-            prefix = prefix + "/"
-        os_snapshot_export_detail = ObjectStorageSnapshotExportDetails(
-            export_type="OBJECT_STORAGE",
-            namespace=namespace,
-            bucket=bucketname,
-            prefix=prefix,
-        )
-
-        snapshot_detail = SnapshotDatasetDetails(
-            are_annotations_included=True,
-            are_unannotated_records_included=include_unlabeled,
-            export_details=os_snapshot_export_detail,
-        )
-        try:
-            snapshot_detail_response = self.dls_cp_client.snapshot_dataset(
-                dataset_id, snapshot_detail
-            )
-        except Exception as error:
-            if dataset_id not in self.list_dataset().index:
-                raise ValueError(
-                    "The parameter `dataset_id` is invalid. "
-                    "Use the `.list_dataset()` method to obtain a list of  all available datasets."
-                )
-            raise error
-
-        res_work_request = self._wait_for_work_request(snapshot_detail_response)
-
-        metadata = res_work_request.data.resources[1].metadata
-        return ObjectStorageDetails(
-            metadata["BUCKET"], metadata["NAMESPACE"], metadata["OBJECT"]
-        ).path
-
-    def _wait_for_work_request(self, snapshot_detail_response):
-        successful_state = WorkRequest.STATUS_SUCCEEDED
-        wait_for_states = (
-            WorkRequest.STATUS_CANCELED,
-            WorkRequest.STATUS_CANCELING,
-            WorkRequest.STATUS_FAILED,
-        )
-        work_request_id = snapshot_detail_response.headers["opc-work-request-id"]
-        work_request_log_entires = self.dls_cp_client.list_work_request_logs(
-            work_request_id
-        ).data.items
-        i = 0
-        res_work_request = self.dls_cp_client.get_work_request(work_request_id)
-        self._num_progress_bar = NUM_PROGRESS_BAR
-        with utils.get_progress_bar(self._num_progress_bar) as progress:
-            while len(work_request_log_entires) <= self._num_progress_bar - 1:
-                new_work_request_log_entires = work_request_log_entires[i:]
-                for work_request_log_entry in new_work_request_log_entires:
-                    progress.update(work_request_log_entry.message)
-                    i += 1
-                if (
-                    len(work_request_log_entires) == self._num_progress_bar - 1
-                    or res_work_request.data.status in wait_for_states
-                ):
-                    progress.update(work_request_log_entires[-1].message)
-                    if res_work_request.data.status != successful_state:
-                        raise Exception(work_request_log_entires[-1].message)
-                    else:
-                        break
-                time.sleep(i)
-                work_request_log_entires = self.dls_cp_client.list_work_request_logs(
-                    work_request_id
-                ).data.items
-                res_work_request = self.dls_cp_client.get_work_request(work_request_id)
-        return res_work_request

ads/data_labeling/interface/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/data_labeling/interface/loader.py

@@ -1,16 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from abc import ABC, abstractmethod
-from typing import Any
-
-
-class Loader(ABC):
-    """Data Loader Interface."""
-
-    @abstractmethod
-    def load(self, **kwargs) -> Any:
-        pass

ads/data_labeling/interface/parser.py

@@ -1,16 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from abc import ABC, abstractmethod
-from typing import Any
-
-
-class Parser(ABC):
-    """Data Parser Interface."""
-
-    @abstractmethod
-    def parse(self) -> Any:
-        pass

ads/data_labeling/interface/reader.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from abc import ABC, abstractmethod
-from typing import Any
-from ads.common.serializer import Serializable
-
-
-class Reader(ABC):
-    """Data Reader Interface."""
-
-    def info(self) -> Serializable:
-        NotImplementedError(
-            f"The class {self.__class__.__name__} did not implement the required method "
-            "`info()`. Contact the class maintainer."
-        )
-
-    @abstractmethod
-    def read(self) -> Any:
-        pass

ads/data_labeling/loader/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/data_labeling/loader/file_loader.py

@@ -1,241 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from io import BytesIO
-from typing import Any, Dict, List, Union
-
-import fsspec
-import PIL
-from ads.common import auth as authutil
-from ads.data_labeling.constants import DatasetType
-from ads.data_labeling.interface.loader import Loader
-from ads.text_dataset.dataset import TextDatasetFactory, backends
-from PIL import Image
-
-THREAD_POOL_MAX_WORKERS = 10
-
-
-class FileLoader:
-    """FileLoader Base Class.
-
-    Attributes:
-    ----------
-    auth: (dict, optional). Defaults to None.
-        The default authetication is set using `ads.set_auth` API. If you need to override the
-        default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-        authentication signer and kwargs required to instantiate IdentityClient object.
-
-    Examples
-    --------
-    >>> from ads.data_labeling.loader.file_loader import FileLoader
-    >>> import oci
-    >>> import os
-    >>> from ads.common import auth as authutil
-    >>> path = "path/to/your_text_file.txt"
-    >>> file_content = FileLoader(auth=authutil.api_keys()).load(path)
-    """
-
-    def __init__(self, auth: Dict = None) -> "FileLoader":
-        """Initiates a FileLoader instance.
-
-        Parameters
-        ----------
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        """
-        self.auth = auth or authutil.default_signer()
-
-    def load(self, path: str, **kwargs) -> BytesIO:
-        """Loads the file content from the path.
-
-        Parameters
-        ----------
-        path: str
-            The file path, can be local or object storage path.
-        kwargs:
-            Nothing.
-
-        Returns
-        -------
-        BytesIO
-            The data in BytesIO format.
-        """
-        data = None
-        try:
-            with fsspec.open(path, **self.auth) as f:
-                data = f.read()
-        except FileNotFoundError:
-            raise FileNotFoundError(f"{path} not found.")
-        except Exception as e:
-            raise e
-
-        return data
-
-    def bulk_load(self, paths: List[str], **kwargs) -> Dict[str, Any]:
-        """Loads the files content from the list of paths.
-        The ThreadPoolExecutor is used to load the files in parallel threads.
-
-        Parameters
-        ----------
-        paths: List[str]
-            The list of file paths, can be local or object storage paths.
-
-        Returns
-        -------
-        Dict[str, Any]
-            The map between file path and file content.
-        """
-        result = {}
-        if not paths or not isinstance(paths, list) or len(paths) == 0:
-            return result
-
-        with ThreadPoolExecutor(max_workers=THREAD_POOL_MAX_WORKERS) as pool:
-            futures = {pool.submit(self.load, path, **kwargs): path for path in paths}
-            result = {futures[task]: task.result() for task in as_completed(futures)}
-        return result
-
-
-class TextFileLoader(FileLoader):
-    """
-    TextFileLoader class which loads text files.
-
-    Examples
-    --------
-    >>> from ads.data_labeling import TextFileLoader
-    >>> import oci
-    >>> import os
-    >>> from ads.common import auth as authutil
-    >>> path = "path/to/your_text_file.txt"
-    >>> file_content = TextFileLoader(auth=authutil.api_keys()).load(path)
-    """
-
-    def load(
-        self, path: str, backend: Union[str, backends.Base] = "default", **kwargs
-    ) -> str:
-        """Loads the content from the path.
-
-        Parameters
-        ----------
-        path: str
-            Text file path, can be local or object storage path.
-        backend: Union[str, backends.Base]
-            Default to "default". Valid options are "default" and "tika" or
-            ads.text_dataset.backends.Base, ads.text_dataset.backends.Tika
-        kwargs:
-            encoding: (str, optional). Defaults to 'utf-8'.
-                Encoding for text files. Used only to extract the content of the text dataset contents.
-
-        Returns
-        -------
-        str
-            Content of the text file.
-        """
-        format = os.path.splitext(path)[1].replace(".", "")
-        try:
-            content = next(
-                TextDatasetFactory.format(format.lower())
-                .backend(backend)
-                .read_text(path, storage_options=self.auth, **kwargs)
-            )[0]
-        except FileNotFoundError:
-            raise FileNotFoundError(f"{path} not found.")
-        except Exception as e:
-            raise e
-        return content
-
-
-class ImageFileLoader(FileLoader):
-    """
-    ImageFileLoader class which loads image files.
-
-    Examples
-    --------
-    >>> from ads.data_labeling import ImageFileLoader
-    >>> import oci
-    >>> import os
-    >>> from ads.common import auth as authutil
-    >>> path = "path/to/image.png"
-    >>> image = ImageFileLoader(auth=authutil.api_keys()).load(path)
-    """
-
-    def load(self, path: str, **kwargs) -> PIL.ImageFile.ImageFile:
-        """Loads the image from the path.
-
-        Parameters
-        ----------
-        path: str
-            Image file path, can be local or object storage path.
-        kwargs:
-            Nothing.
-
-        Returns
-        -------
-        PIL.ImageFile.ImageFile
-            Image opened by Pillow.
-        """
-        data = None
-        data = super().load(path=path)
-
-        return Image.open(BytesIO(data))
-
-
-class FileLoaderFactory:
-    """FileLoaderFactory class to create/register FileLoaders."""
-
-    _loaders = {
-        DatasetType.TEXT: TextFileLoader,
-        DatasetType.IMAGE: ImageFileLoader,
-        DatasetType.DOCUMENT: FileLoader,
-    }
-
-    @staticmethod
-    def loader(dataset_type: str, auth: Dict = None) -> FileLoader:
-        """Gets the loader based on the dataset_type.
-
-        Parameters
-        ----------
-        dataset_type: str
-            Dataset type. Currently supports TEXT, IMAGE and DOCUMENT.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-
-        Returns
-        -------
-        FileLoader
-            A FileLoader instance corresponding to the dataset_type.
-        """
-        if not dataset_type in FileLoaderFactory._loaders:
-            raise ValueError(
-                f"The wrong dataset type has been provided. "
-                f"Supported dataset types are: `{DatasetType.TEXT}`, "
-                f"`{DatasetType.IMAGE}` and `{DatasetType.DOCUMENT}`."
-            )
-
-        return FileLoaderFactory._loaders[dataset_type](auth=auth)
-
-    @classmethod
-    def register(cls, dataset_type: str, loader: Loader) -> None:
-        """Registers a new loader for a given dataset_type.
-
-        Parameters
-        ----------
-        dataset_type: str
-            Dataset type. Currently supports TEXT and IMAGE.
-        loader: Loader
-            A Loader class which supports loading content of the given dataset_type.
-
-        Returns
-        -------
-        None
-            Nothing.
-        """
-        cls._parsers[dataset_type] = loader

ads/data_labeling/metadata.py

@@ -1,110 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from dataclasses import asdict, dataclass, field
-from typing import Any, Dict, List
-
-import pandas as pd
-from ads.common.object_storage_details import ObjectStorageDetails
-from ads.common.serializer import DataClassSerializable
-from oci.data_labeling_service_dataplane.models.dataset import Dataset as OCIDLSDataset
-
-
-@dataclass
-class Metadata(DataClassSerializable):
-    """The class that representing the labeled dataset metadata.
-
-    Attributes
-    ----------
-    source_path: str
-        Contains information on where all the source data(image/text/document) stores.
-    records_path: str
-        Contains information on where records jsonl file stores.
-    labels: List
-        List of classes/labels for the dataset.
-    dataset_name: str
-        Dataset display name on the Data Labeling Service console.
-    compartment_id: str
-        Compartment id of the labeled dataset.
-    dataset_id: str
-        Dataset id.
-    annotation_type: str
-        Type of the labeling/annotation task. Currently supports SINGLE_LABEL,
-        MULTI_LABEL, ENTITY_EXTRACTION, BOUNDING_BOX.
-    dataset_type: str
-        Type of the dataset. Currently supports Text, Image, DOCUMENT.
-    """
-
-    source_path: str = ""
-    records_path: str = ""
-    labels: List[str] = field(default_factory=list)
-    dataset_name: str = ""
-    compartment_id: str = ""
-    dataset_id: str = ""
-    annotation_type: str = ""
-    dataset_type: str = ""
-
-    def to_dict(self) -> Dict:
-        """Converts to dictionary representation.
-
-        Returns
-        -------
-        Dict
-            The metadata in dictionary type.
-        """
-        return asdict(self)
-
-    def __repr__(self):
-        """Show the Metadata in yaml format."""
-        return self.to_yaml()
-
-    def to_dataframe(self) -> pd.DataFrame:
-        """
-        Converts the metadata to dataframe format.
-
-        Returns
-        -------
-        pandas.DataFrame
-            The metadata in Pandas dataframe format.
-        """
-        return pd.DataFrame({"": self.to_dict()})
-
-    def _repr_html_(self):
-        """Shows metadata in dataframe format."""
-        return (
-            self.to_dataframe().style.set_properties(**{"margin-left": "0px"}).to_html()
-        )
-
-    @classmethod
-    def from_dls_dataset(cls, dataset: OCIDLSDataset) -> "Metadata":
-        """Contructs a Metadata instance from OCI DLS dataset.
-
-        Parameters
-        ----------
-        dataset: OCIDLSDataset
-            OCIDLSDataset object.
-
-        Returns
-        -------
-        Metadata
-            The ads labeled dataset metadata instance.
-        """
-        oci_labels = [d.name for d in dataset.label_set.items]
-        oci_source_path = ObjectStorageDetails(
-            dataset.dataset_source_details.bucket,
-            dataset.dataset_source_details.namespace,
-            dataset.dataset_source_details.prefix,
-        ).path
-
-        return Metadata(
-            annotation_type=dataset.annotation_format,
-            dataset_name=dataset.display_name,
-            source_path=oci_source_path,
-            labels=oci_labels,
-            compartment_id=dataset.compartment_id,
-            dataset_id=dataset.id,
-            dataset_type=dataset.dataset_format_details.format_type,
-        )

ads/data_labeling/mixin/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/