oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/data_labeling/reader/dataset_reader.py

@@ -1,574 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-"""
-The module containing classes to read labeled datasets.
-Allows to read labeled datasets from exports or from the cloud.
-
-Classes
--------
-    LabeledDatasetReader
-        The LabeledDatasetReader class to read labeled dataset.
-    ExportReader
-        The ExportReader class to read labeled dataset from the export.
-    DLSDatasetReader
-        The DLSDatasetReader class to read labeled dataset from the cloud.
-
-Examples
---------
-    >>> from ads.common import auth as authutil
-    >>> from ads.data_labeling import LabeledDatasetReader
-    >>> ds_reader = LabeledDatasetReader.from_export(
-    ...    path="oci://bucket_name@namespace/dataset_metadata.jsonl",
-    ...    auth=authutil.api_keys(),
-    ...    materialize=True
-    ... )
-    >>> ds_reader.info()
-        ------------------------------------------------------------------------
-        annotation_type	                                            SINGLE_LABEL
-        compartment_id	                                        TEST_COMPARTMENT
-        dataset_id                                                  TEST_DATASET
-        dataset_name                                           test_dataset_name
-        dataset_type                                                        TEXT
-        labels	                                                   ['yes', 'no']
-        records_path                                             path/to/records
-        source_path                                              path/to/dataset
-
-    >>> ds_reader.read()
-                                 Path            Content            Annotations
-        -----------------------------------------------------------------------
-        0   path/to/the/content/file1       file content                    yes
-        1   path/to/the/content/file2       file content                     no
-        2   path/to/the/content/file3       file content                     no
-
-    >>> next(ds_reader.read(iterator=True))
-        ("path/to/the/content/file1", "file content", "yes")
-
-    >>> next(ds_reader.read(iterator=True, chunksize=2))
-        [("path/to/the/content/file1", "file content", "yes"),
-        ("path/to/the/content/file2", "file content", "no")]
-
-    >>> next(ds_reader.read(chunksize=2))
-                                Path            Content            Annotations
-        ----------------------------------------------------------------------
-        0   path/to/the/content/file1       file content                    yes
-        1   path/to/the/content/file2       file content                     no
-
-    >>> ds_reader = LabeledDatasetReader.from_DLS(
-    ...    dataset_id="dataset_OCID",
-    ...    compartment_id="compartment_OCID",
-    ...    auth=authutil.api_keys(),
-    ...    materialize=True
-    ... )
-"""
-
-from functools import lru_cache
-from typing import Any, Dict, Generator, Tuple, Union
-
-import pandas as pd
-from ads.common import auth as authutil
-from ads.common.serializer import Serializable
-from ads.data_labeling.interface.reader import Reader
-from ads.data_labeling.reader.metadata_reader import Metadata, MetadataReader
-from ads.data_labeling.reader.record_reader import RecordReader
-from ads.data_labeling.constants import (
-    FORMATS_TO_ANNOTATION_TYPE,
-    ANNOTATION_TYPE_TO_FORMATS,
-)
-from ads.config import NB_SESSION_COMPARTMENT_OCID, JOB_RUN_COMPARTMENT_OCID
-
-_LABELED_DF_COLUMNS = ["Path", "Content", "Annotations"]
-
-
-class LabeledDatasetReader:
-    """The labeled dataset reader class.
-
-    Methods
-    -------
-    info(self) -> Metadata
-        Gets labeled dataset metadata.
-    read(self, iterator: bool = False) -> Union[Generator[Any, Any, Any], pd.DataFrame]
-        Reads labeled dataset.
-    from_export(cls, path: str, auth: Dict = None, encoding="utf-8", materialize: bool = False) -> "LabeledDatasetReader"
-        Constructs a Labeled Dataset Reader instance.
-
-    Examples
-    --------
-    >>> from ads.common import auth as authutil
-    >>> from ads.data_labeling import LabeledDatasetReader
-
-    >>> ds_reader = LabeledDatasetReader.from_export(
-    ...    path="oci://bucket_name@namespace/dataset_metadata.jsonl",
-    ...    auth=authutil.api_keys(),
-    ...    materialize=True
-    ... )
-
-    >>> ds_reader = LabeledDatasetReader.from_DLS(
-    ...    dataset_id="dataset_OCID",
-    ...    compartment_id="compartment_OCID",
-    ...    auth=authutil.api_keys(),
-    ...    materialize=True
-    ... )
-
-    >>> ds_reader.info()
-        ------------------------------------------------------------------------
-        annotation_type	                                            SINGLE_LABEL
-        compartment_id	                                        TEST_COMPARTMENT
-        dataset_id                                                  TEST_DATASET
-        dataset_name                                           test_dataset_name
-        dataset_type                                                        TEXT
-        labels	                                                   ['yes', 'no']
-        records_path                                             path/to/records
-        source_path                                              path/to/dataset
-
-    >>> ds_reader.read()
-                                 Path            Content            Annotations
-        -----------------------------------------------------------------------
-        0   path/to/the/content/file1       file content                    yes
-        1   path/to/the/content/file2       file content                     no
-        2   path/to/the/content/file3       file content                     no
-
-    >>> next(ds_reader.read(iterator=True))
-        ("path/to/the/content/file1", "file content", "yes")
-
-    >>> next(ds_reader.read(iterator=True, chunksize=2))
-        [("path/to/the/content/file1", "file content", "yes"),
-        ("path/to/the/content/file2", "file content", "no")]
-
-    >>> next(ds_reader.read(chunksize=2))
-                                Path            Content            Annotations
-        ----------------------------------------------------------------------
-        0   path/to/the/content/file1       file content                    yes
-        1   path/to/the/content/file2       file content                     no
-    """
-
-    def __init__(self, reader: Reader):
-        """Initializes the labeled dataset reader instance.
-
-        Parameters
-        ----------
-        reader: Reader
-            The Reader instance which reads and extracts the labeled dataset.
-        """
-        self._reader = reader
-
-    @classmethod
-    def from_export(
-        cls,
-        path: str,
-        auth: dict = None,
-        encoding: str = "utf-8",
-        materialize: bool = False,
-        include_unlabeled: bool = False,
-    ) -> "LabeledDatasetReader":
-        """Constructs Labeled Dataset Reader instance.
-
-        Parameters
-        ----------
-        path: str
-            The metadata file path, can be either local or object storage path.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        encoding: (str, optional). Defaults to 'utf-8'.
-            Encoding for files.
-        materialize: (bool, optional). Defaults to False.
-            Whether the content of the dataset file should be loaded or it should return the file path to the content.
-            By default the content will not be loaded.
-
-        Returns
-        -------
-        LabeledDatasetReader
-            The LabeledDatasetReader instance.
-        """
-        auth = auth or authutil.default_signer()
-
-        return cls(
-            reader=ExportReader(
-                path=path,
-                auth=auth,
-                encoding=encoding,
-                materialize=materialize,
-                include_unlabeled=include_unlabeled,
-            )
-        )
-
-    @classmethod
-    def from_DLS(
-        cls,
-        dataset_id: str,
-        compartment_id: str = None,
-        auth: dict = None,
-        encoding: str = "utf-8",
-        materialize: bool = False,
-        include_unlabeled: bool = False,
-    ) -> "LabeledDatasetReader":
-        """Constructs Labeled Dataset Reader instance.
-
-        Parameters
-        ----------
-        dataset_id: str
-            The dataset OCID.
-        compartment_id: str. Defaults to the compartment_id from the env variable.
-            The compartment OCID of the dataset.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        encoding: (str, optional). Defaults to 'utf-8'.
-            Encoding for files.
-        materialize: (bool, optional). Defaults to False.
-            Whether the content of the dataset file should be loaded or it should return the file path to the content.
-            By default the content will not be loaded.
-
-        Returns
-        -------
-        LabeledDatasetReader
-            The LabeledDatasetReader instance.
-        """
-        if compartment_id is None:
-            compartment_id = NB_SESSION_COMPARTMENT_OCID or JOB_RUN_COMPARTMENT_OCID
-
-        if not compartment_id:
-            raise ValueError("The `compartment_id` must be provided.")
-
-        return cls(
-            reader=DLSDatasetReader(
-                compartment_id=compartment_id,
-                dataset_id=dataset_id,
-                auth=auth or authutil.default_signer(),
-                encoding=encoding,
-                materialize=materialize,
-                include_unlabeled=include_unlabeled,
-            )
-        )
-
-    def info(self) -> Serializable:
-        """Gets the labeled dataset metadata.
-
-        Returns
-        -------
-        Metadata
-            The labeled dataset metadata.
-        """
-        return self._reader.info()
-
-    def read(
-        self, iterator: bool = False, format: str = None, chunksize: int = None
-    ) -> Union[Generator[Any, Any, Any], pd.DataFrame]:
-        """Reads the labeled dataset records.
-
-        Parameters
-        ----------
-        iterator: (bool, optional). Defaults to False.
-            True if the result should be represented as a Generator.
-            Fasle if the result should be represented as a Pandas DataFrame.
-        format: (str, optional). Defaults to None.
-            Output format of annotations. Can be None, "spacy" or "yolo".
-        chunksize: (int, optional). Defaults to None.
-            The number of records that should be read in one iteration.
-            The result will be returned in a generator format.
-
-        Returns
-        -------
-        Union[
-            Generator[Tuple[str, str, Any], Any, Any],
-            Generator[List[Tuple[str, str, Any]], Any, Any],
-            Generator[pd.DataFrame, Any, Any],
-            pd.DataFrame
-        ]
-            `pd.Dataframe` if `iterator` and `chunksize` are not specified.
-            `Generator[pd.Dataframe] ` if `iterator` equal to False and `chunksize` is specified.
-            `Generator[List[Tuple[str, str, Any]]]` if `iterator` equal to True and `chunksize` is specified.
-            `Generator[Tuple[str, str, Any]]` if `iterator` equal to True and `chunksize` is not specified.
-        """
-
-        if chunksize:
-            return self._bulk_read(
-                iterator=iterator, format=format, chunksize=chunksize
-            )
-
-        if iterator:
-            return self._reader.read(format=format)
-
-        return pd.DataFrame(
-            self._reader.read(format=format), columns=_LABELED_DF_COLUMNS
-        )
-
-    def _bulk_read(
-        self, iterator: bool = False, format: str = None, chunksize: int = None
-    ) -> Generator[Union[pd.DataFrame, Tuple[str, str, Any]], Any, Any]:
-        """Reads the labeled dataset records by chunks.
-
-        Parameters
-        ----------
-        iterator: (bool, optional). Defaults to False.
-            True if the result should be represented as a Generator.
-            Fasle if the result should be represented as a Pandas DataFrame.
-        format: (str, optional). Defaults to None.
-            Output format of annotations. Can be None, "spacy" or "yolo".
-        chunksize: (int, optional). Defaults to None.
-            The number of records that should be read in one iteration.
-            Result will be represented as a generator.
-
-        Yields
-        -------
-        Generator[Union[pd.DataFrame, Tuple[str, str, Any]], Any, Any]
-            The generator that yields records either in Dataframe format or Tuple.
-
-        Raises
-        ------
-            ValueError: If chunksize is empty or not a positive integer.
-        """
-        if not chunksize or not isinstance(chunksize, int) or chunksize < 1:
-            raise ValueError("`chunksize` must be a positive integer.")
-
-        result = []
-        i = 0
-        for record in self._reader.read(format=format):
-            result.append(record)
-            i += 1
-            if i >= chunksize:
-                yield result if iterator else pd.DataFrame(
-                    result, columns=_LABELED_DF_COLUMNS
-                )
-                result = []
-                i = 0
-        if result:
-            yield result if iterator else pd.DataFrame(
-                result, columns=_LABELED_DF_COLUMNS
-            )
-
-
-class DLSDatasetReader(Reader):
-    """The DLSDatasetReader class to read labeled dataset from the cloud.
-
-    Methods
-    -------
-    info(self) -> Metadata
-        Gets the labeled dataset metadata.
-    read(self) -> Generator[Tuple, Any, Any]
-        Reads the labeled dataset.
-    """
-
-    def __init__(
-        self,
-        dataset_id: str,
-        compartment_id: str,
-        auth: Dict,
-        encoding="utf-8",
-        materialize: bool = False,
-        include_unlabeled: bool = False,
-    ):
-        """Initializes the DLS dataset reader instance.
-
-        Parameters
-        ----------
-        dataset_id: str
-            The dataset OCID.
-        compartment_id: str
-            The compartment OCID of the dataset.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        encoding: (str, optional). Defaults to 'utf-8'.
-            Encoding for files. The encoding is used to extract the metadata information
-            of the labeled dataset and also to extract the content of the text dataset records.
-        materialize: (bool, optional). Defaults to False.
-            Whether the content of dataset files should be loaded/materialized or not.
-            By default the content will not be materialized.
-        include_unlabeled: (bool, optional). Defaults to False.
-            Whether to load the unlabeled records or not.
-
-        Raises
-        ------
-            ValueError: When dataset_id is empty or not a string.
-            TypeError: When dataset_id not a string.
-        """
-        if not dataset_id:
-            raise ValueError("The dataset OCID must be specified.")
-
-        if not isinstance(dataset_id, str):
-            raise TypeError("The dataset_id must be a string.")
-
-        if not compartment_id:
-            raise ValueError("The compartment OCID must be specified.")
-
-        self.dataset_id = dataset_id
-        self.compartment_id = compartment_id
-        self.auth = auth or authutil.default_signer()
-        self.encoding = encoding
-        self.materialize = materialize
-        self.include_unlabeled = include_unlabeled
-
-    @lru_cache(maxsize=1)
-    def info(self) -> Metadata:
-        """Gets the labeled dataset metadata.
-
-        Returns
-        -------
-        Metadata
-            The labeled dataset metadata.
-        """
-        return MetadataReader.from_DLS(
-            compartment_id=self.compartment_id,
-            dataset_id=self.dataset_id,
-        ).read()
-
-    def read(self, format: str = None) -> Generator[Tuple, Any, Any]:
-        """Reads the labeled dataset records.
-
-        Parameters
-        ----------
-        format: (str, optional). Defaults to None.
-            Output format of annotations. Can be None, "spacy" for dataset
-            Entity Extraction type or "yolo" for Object Detection type.
-            When None, it outputs List[NERItem] or List[BoundingBoxItem].
-            When "spacy", it outputs List[Tuple].
-            When "yolo", it outputs List[List[Tuple]].
-
-        Returns
-        -------
-        Generator[Tuple, Any, Any]
-            The labeled dataset records.
-        """
-
-        metadata = self.info()
-
-        records_reader = RecordReader.from_DLS(
-            dataset_type=metadata.dataset_type,
-            annotation_type=metadata.annotation_type,
-            dataset_source_path=metadata.source_path,
-            compartment_id=self.compartment_id,
-            dataset_id=self.dataset_id,
-            auth=self.auth,
-            encoding=self.encoding,
-            materialize=self.materialize,
-            include_unlabeled=self.include_unlabeled,
-            format=format,
-            categories=metadata.labels,
-        )
-        return records_reader.read()
-
-
-class ExportReader(Reader):
-    """The ExportReader class to read labeled dataset from the export.
-
-    Methods
-    -------
-    info(self) -> Metadata
-        Gets the labeled dataset metadata.
-    read(self) -> Generator[Tuple, Any, Any]
-        Reads the labeled dataset.
-    """
-
-    def __init__(
-        self,
-        path: str,
-        auth: Dict = None,
-        encoding="utf-8",
-        materialize: bool = False,
-        include_unlabeled: bool = False,
-    ):
-        """Initializes the labeled dataset export reader instance.
-
-        Parameters
-        ----------
-        path: str
-            The metadata file path, can be either local or object storage path.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        encoding: (str, optional). Defaults to 'utf-8'.
-            Encoding for files. The encoding is used to extract the metadata information
-            of the labeled dataset and also to extract the content of the text dataset records.
-        materialize: (bool, optional). Defaults to False.
-            Whether the content of dataset files should be loaded/materialized or not.
-            By default the content will not be materialized.
-        include_unlabeled: (bool, optional). Defaults to False.
-            Whether to load the unlabeled records or not.
-
-        Raises
-        ------
-            ValueError: When path is empty or not a string.
-            TypeError: When path not a string.
-        """
-
-        if not path:
-            raise ValueError("The parameter `path` is required.")
-
-        if not isinstance(path, str):
-            raise TypeError("The parameter `path` must be a string.")
-
-        self.path = path
-        self.auth = auth or authutil.default_signer()
-        self.encoding = encoding
-        self.materialize = materialize
-        self.include_unlabeled = include_unlabeled
-
-    @lru_cache(maxsize=1)
-    def info(self) -> Metadata:
-        """Gets the labeled dataset metadata.
-
-        Returns
-        -------
-        Metadata
-            The labeled dataset metadata.
-        """
-        return MetadataReader.from_export_file(
-            path=self.path,
-            auth=self.auth,
-        ).read()
-
-    def read(self, format: str = None) -> Generator[Tuple, Any, Any]:
-        """Reads the labeled dataset records.
-
-        Parameters
-        ----------
-        format: (str, optional). Defaults to None.
-            Output format of annotations. Can be None, "spacy" for dataset
-            Entity Extraction type or "yolo" for Object Detection type.
-            When None, it outputs List[NERItem] or List[BoundingBoxItem].
-            When "spacy", it outputs List[Tuple].
-            When "yolo", it outputs List[List[Tuple]].
-
-        Returns
-        -------
-        Generator[Tuple, Any, Any]
-            The labeled dataset records.
-        """
-        metadata = self.info()
-        if (
-            format
-            and isinstance(format, str)
-            and (
-                format.lower() not in FORMATS_TO_ANNOTATION_TYPE.keys()
-                or FORMATS_TO_ANNOTATION_TYPE[format.lower()]
-                != metadata.annotation_type
-            )
-        ):
-            raise TypeError(
-                "Wrong format. `format` can only be None or "
-                f"`{ANNOTATION_TYPE_TO_FORMATS[metadata.annotation_type]}`."
-            )
-
-        records_reader = RecordReader.from_export_file(
-            path=metadata.records_path or self.path,
-            dataset_type=metadata.dataset_type,
-            annotation_type=metadata.annotation_type,
-            dataset_source_path=metadata.source_path,
-            auth=self.auth,
-            encoding=self.encoding,
-            materialize=self.materialize,
-            include_unlabeled=self.include_unlabeled,
-            format=format,
-            categories=metadata.labels,
-            includes_metadata=not metadata.records_path,
-        )
-        return records_reader.read()

ads/data_labeling/reader/dls_record_reader.py

@@ -1,121 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from collections import defaultdict
-from dataclasses import dataclass
-from typing import Any, Generator, List
-
-from ads.common import auth as authutil
-from ads.common import oci_client
-from ads.data_labeling.interface.reader import Reader
-from oci.data_labeling_service_dataplane.models import AnnotationSummary, RecordSummary
-from oci.exceptions import ServiceError
-from oci.pagination import list_call_get_all_results
-
-
-class ReadRecordsError(Exception):   # pragma: no cover
-    def __init__(self, dataset_id: str):
-        super().__init__(
-            f"Error occurred in attempt to read records of dataset {dataset_id}."
-        )
-
-
-class ReadAnnotationsError(Exception):   # pragma: no cover
-    def __init__(self, dataset_id: str):
-        super().__init__(
-            f"Error occurred in attempt to read annotations of dataset {dataset_id}."
-        )
-
-
-@dataclass
-class OCIRecordSummary:
-    """The class that representing the labeled record in ADS format.
-
-    Attributes
-    ----------
-    record: RecordSummary
-        OCI RecordSummary.
-    annotations: List[AnnotationSummary]
-        List of OCI AnnotationSummary.
-    """
-
-    record: RecordSummary = None
-    annotation: List[AnnotationSummary] = None
-
-
-class DLSRecordReader(Reader):
-    """DLS Record Reader Class that reads records from the cloud into ADS format."""
-
-    def __init__(self, dataset_id: str, compartment_id: str, auth: dict = None):
-        """Initiates a DLSRecordReader instance.
-
-        Parameters
-        ----------
-        dataset_id: str
-            The dataset OCID.
-        compartment_id: str
-            The compartment OCID of the dataset.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        """
-        if not dataset_id:
-            raise ValueError("The dataset OCID must be specified.")
-
-        if not isinstance(dataset_id, str):
-            raise TypeError("The dataset_id must be a string.")
-
-        if not compartment_id:
-            raise ValueError("The compartment OCID must be specified.")
-
-        if not isinstance(compartment_id, str):
-            raise TypeError("The compartment OCID must be a string.")
-
-        auth = auth or authutil.default_signer()
-        self.dataset_id = dataset_id
-        self.compartment_id = compartment_id
-        self.dls_dp_client = oci_client.OCIClientFactory(**auth).data_labeling_dp
-
-    def _read_records(self):
-        try:
-            return list_call_get_all_results(
-                self.dls_dp_client.list_records,
-                self.compartment_id,
-                self.dataset_id,
-                lifecycle_state="ACTIVE",
-            ).data
-        except ServiceError:
-            raise ReadRecordsError(self.dataset_id)
-
-    def _read_annotations(self):
-        try:
-            return list_call_get_all_results(
-                self.dls_dp_client.list_annotations,
-                self.compartment_id,
-                self.dataset_id,
-                lifecycle_state="ACTIVE",
-            ).data
-        except ServiceError:
-            raise ReadAnnotationsError(self.dataset_id)
-
-    def read(self) -> Generator[OCIRecordSummary, Any, Any]:
-        """Reads OCI records.
-
-        Yields
-        ------
-        OCIRecordSummary
-            The OCIRecordSummary instance.
-        """
-        records = self._read_records()
-        annotations = self._read_annotations()
-
-        annotations_map = defaultdict(list)
-        for annotation in annotations:
-            annotations_map[annotation.record_id].append(annotation)
-
-        for record in records:
-            yield OCIRecordSummary(record, annotations_map.get(record.id))