oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/data_labeling/reader/export_record_reader.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import Any, Dict, Generator
+from ads.data_labeling.reader.jsonl_reader import JsonlReader
+
+
+class ExportRecordReader(JsonlReader):
+    """The ExportRecordReader class to read labeled dataset records from the export.
+
+    Methods
+    -------
+    read(self) -> Generator[Dict, Any, Any]
+        Reads labeled dataset records.
+    """
+
+    def __init__(
+        self,
+        path: str,
+        auth: Dict = None,
+        encoding="utf-8",
+        includes_metadata: bool = False,
+    ) -> "ExportRecordReader":
+        """Initiates an ExportRecordReader instance.
+
+        Parameters
+        ----------
+        path: str
+            object storage path or local path for a file.
+        auth: (dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+        encoding : (str, optional). Defaults to 'utf-8'.
+            Encoding of files. Only used for "TEXT" dataset.
+        includes_metadata: (bool, optional). Defaults to False.
+            Determines whether the export file includes metadata or not.
+
+        Examples
+        --------
+        >>> from ads.data_labeling.reader.export_record_reader import ExportRecordReader
+        >>> path = "your/path/to/jsonl/file.jsonl"
+        >>> from ads.common import auth as authutil
+        >>> reader = ExportRecordReader(path=path, auth=authutil.api_keys(), encoding="utf-8")
+        >>> next(reader.read())
+        """
+        super().__init__(path=path, auth=auth, encoding=encoding)
+        self._includes_metadata = includes_metadata
+
+    def read(self) -> Generator[Dict, Any, Any]:
+        """Reads labeled dataset records.
+
+        Returns
+        -------
+        Generator[Dict, Any, Any]
+            The labeled dataset records.
+        """
+        skip = 1 if self._includes_metadata else None
+        return super().read(skip=skip)

ads/data_labeling/reader/jsonl_reader.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import json
+from typing import Any, Dict, Generator
+
+import fsspec
+from ads.data_labeling.interface.reader import Reader
+from ads.common import auth as authutil
+
+
+class JsonlReader(Reader):
+    """JsonlReader class which reads the file."""
+
+    def __init__(self, path: str, auth: Dict = None, encoding="utf-8") -> "JsonlReader":
+        """Initiates a JsonlReader object.
+
+        Parameters
+        ----------
+        path: str
+            object storage path or local path for a file.
+        auth: (dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+        encoding : (str, optional). Defaults to 'utf-8'.
+            Encoding of files. Only used for "TEXT" dataset.
+
+        Examples
+        --------
+        >>> from ads.data_labeling.reader.jsonl_reader import JsonlReader
+        >>> path = "your/path/to/jsonl/file.jsonl"
+        >>> from ads.common import auth as authutil
+        >>> reader = JsonlReader(path=path, auth=authutil.api_keys(), encoding="utf-8")
+        >>> next(reader.read())
+        """
+        self.path = path
+        self.auth = auth or authutil.default_signer()
+        self.encoding = encoding
+
+    def read(self, skip: int = None) -> Generator[Dict, Any, Any]:
+        """Reads and yields the content of the file.
+
+        Parameters
+        ----------
+        skip: (int, optional). Defaults to None.
+            The number of records that should be skipped.
+
+        Returns
+        -------
+        Generator[Dict, Any, Any]
+            The content of the file.
+
+        Raises
+        ------
+        ValueError
+            If `skip` not empty and not a positive integer.
+        FileNotFoundError
+            When file not found.
+        """
+        if skip and (not isinstance(skip, int) or skip < 1):
+            raise ValueError("The parameter `skip` must be a positive integer.")
+        try:
+            line_number = 0
+            with fsspec.open(self.path, "r", encoding=self.encoding, **self.auth) as f:
+                for line in f:
+                    line_number += 1
+                    if skip and line_number <= skip:
+                        continue
+                    yield json.loads(line)
+        except FileNotFoundError:
+            raise FileNotFoundError(f"Path ({self.path}) not found.")

ads/data_labeling/reader/metadata_reader.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import Dict
+
+from ads.common import auth as auth_util
+from ads.common import oci_client
+from ads.config import JOB_RUN_COMPARTMENT_OCID, NB_SESSION_COMPARTMENT_OCID
+from ads.data_labeling.interface.reader import Reader
+from ads.data_labeling.metadata import Metadata
+from ads.data_labeling.parser.export_metadata_parser import MetadataParser
+from ads.data_labeling.reader.jsonl_reader import JsonlReader
+from oci.exceptions import ServiceError
+
+
+class EmptyMetadata(Exception):   # pragma: no cover
+    """Empty Metadata."""
+
+    pass
+
+
+class ReadDatasetError(Exception):   # pragma: no cover
+    def __init__(self, id: str):
+        super().__init__(f"Error occurred in attempt to read dataset {id}.")
+
+
+class DatasetNotFoundError(Exception):   # pragma: no cover
+    def __init__(self, id: str):
+        super().__init__(f"{id} not found.")
+
+
+class MetadataReader:
+    """MetadataReader class which reads and extracts the labeled dataset metadata.
+
+    Examples
+    --------
+    >>> from ads.data_labeling import MetadataReader
+    >>> import oci
+    >>> import os
+    >>> from ads.common import auth as authutil
+    >>> reader = MetadataReader.from_export_file("metadata_export_file_path",
+    ...                                 auth=authutil.api_keys())
+    >>> reader.read()
+    """
+
+    def __init__(self, reader: Reader):
+        """Initiate a MetadataReader instance.
+
+        Parameters
+        ----------
+        reader: Reader
+            Reader instance which reads and extracts the labeled dataset metadata.
+        """
+        self._reader = reader
+
+    @classmethod
+    def from_export_file(cls, path: str, auth: Dict = None) -> "MetadataReader":
+        """Contructs a MetadataReader instance.
+
+        Parameters
+        ----------
+        path: str
+            metadata file path, can be either local or object storage path.
+        auth: (dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+
+        Returns
+        -------
+        MetadataReader
+            The MetadataReader instance whose reader is a ExportMetadataReader instance.
+        """
+        return cls(ExportMetadataReader(path=path, auth=auth))
+
+    @classmethod
+    def from_DLS(
+        cls, dataset_id: str, compartment_id: str = None, auth: dict = None
+    ) -> "MetadataReader":
+        """Contructs a MetadataReader instance.
+
+        Parameters
+        ----------
+        dataset_id: str
+            The dataset OCID.
+        compartment_id: (str, optional). Default None
+            The compartment OCID of the dataset.
+        auth: (dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+
+        Returns
+        -------
+        MetadataReader
+            The MetadataReader instance whose reader is a DLSMetadataReader instance.
+        """
+        if compartment_id is None:
+            compartment_id = NB_SESSION_COMPARTMENT_OCID or JOB_RUN_COMPARTMENT_OCID
+
+        if not compartment_id:
+            raise ValueError("The `compartment_id` must be provided.")
+
+        auth = auth or auth_util.default_signer()
+
+        return cls(
+            DLSMetadataReader(
+                dataset_id=dataset_id, compartment_id=compartment_id, auth=auth
+            )
+        )
+
+    def read(self) -> Metadata:
+        """Reads the content from the metadata file.
+
+        Returns
+        -------
+        Metadata
+            The metadata of the labeled dataset.
+        """
+        return self._reader.read()
+
+
+class DLSMetadataReader(Reader):
+    """DLSMetadataReader class which reads the metadata jsonl file from the cloud."""
+
+    def __init__(self, dataset_id: str, compartment_id: str, auth: dict):
+        """Initializes the DLS metadata reader instance.
+
+        Parameters
+        ----------
+        dataset_id: str
+            The dataset OCID.
+        compartment_id: str
+            The compartment OCID of the dataset.
+        auth: dict
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+
+        Raises
+        ------
+            ValueError: When dataset_id is empty or not a string.
+            TypeError: When dataset_id not a string.
+        """
+        if not dataset_id:
+            raise ValueError("The dataset OCID must be specified.")
+
+        if not isinstance(dataset_id, str):
+            raise TypeError("The dataset_id must be a string.")
+        self.dataset_id = dataset_id
+        self.compartment_id = compartment_id
+
+        self.dls_dp_client = oci_client.OCIClientFactory(**auth).data_labeling_dp
+
+    def read(self) -> Metadata:
+        """Reads the content from the metadata file.
+
+        Returns
+        -------
+        Metadata
+            The metadata of the labeled dataset.
+
+        Raises
+        ------
+        DatasetNotFoundError
+            If dataset not found.
+        ReadDatasetError
+            If any error occured in attempt to read dataset.
+        """
+        try:
+            dataset_response = self.dls_dp_client.get_dataset(self.dataset_id)
+        except ServiceError as service_err:
+            if service_err.status == 404:
+                raise DatasetNotFoundError(self.dataset_id)
+            raise ReadDatasetError(self.dataset_id)
+        return Metadata.from_dls_dataset(dataset_response.data)
+
+
+class ExportMetadataReader(JsonlReader):
+    """ExportMetadataReader class which reads the metadata jsonl file from local/object
+    storage path."""
+
+    def read(self) -> Metadata:
+        """Reads the content from the metadata file.
+
+        Returns
+        -------
+        Metadata
+            The metadata of the labeled dataset.
+        """
+
+        try:
+            return MetadataParser.parse(next(super().read()))
+        except StopIteration:
+            raise EmptyMetadata(
+                "The dataset metadata file is invalid. It appears to be empty. "
+                "Use the `DataLabeling.export()` method to create a new dataset metadata file."
+            )
+        except Exception as e:
+            raise e

ads/data_labeling/reader/record_reader.py

@@ -0,0 +1,263 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import Any, Dict, Generator, List, Tuple, Union
+
+from ads.config import JOB_RUN_COMPARTMENT_OCID, NB_SESSION_COMPARTMENT_OCID
+from ads.data_labeling.interface.loader import Loader
+from ads.data_labeling.interface.parser import Parser
+from ads.data_labeling.interface.reader import Reader
+from ads.data_labeling.loader.file_loader import FileLoaderFactory
+from ads.data_labeling.parser.dls_record_parser import DLSRecordParserFactory
+from ads.data_labeling.parser.export_record_parser import RecordParserFactory
+from ads.data_labeling.reader.dls_record_reader import DLSRecordReader
+from ads.data_labeling.reader.export_record_reader import ExportRecordReader
+
+
+class RecordReader:
+    """Record Reader Class consists of parser, reader and loader. Reader reads the
+    the content from the record file. Parser parses the label for each record. And
+    Loader loads the content of the file path in that record.
+
+    Examples
+    --------
+    >>> import os
+    >>> import oci
+    >>> from ads.data_labeling import RecordReader
+    >>> from ads.common import auth as authutil
+    >>> file_path = "/path/to/your_record.jsonl"
+    >>> dataset_type = "IMAGE"
+    >>> annotation_type = "BOUNDING_BOX"
+    >>> record_reader = RecordReader.from_export_file(file_path, dataset_type, annotation_type, "image_file_path", authutil.api_keys())
+    >>> next(record_reader.read())
+    """
+
+    def __init__(
+        self,
+        reader: Reader,
+        parser: Parser,
+        loader: Loader = None,
+        include_unlabeled: bool = False,
+        encoding: str = "utf-8",
+        materialize: bool = False,
+    ) -> "RecordReader":
+        """Initiates a RecordReader instance.
+
+        Parameters
+        ----------
+        reader: Reader
+            Reader instance to read content from the record file.
+        parser: Parser
+            Parser instance to parse the labels from record file.
+        loader: Loader. Defaults to None.
+            Loader instance to load the content from the file path in the record.
+        materialize: bool, optional. Defaults to False.
+            Whether to materialize the content using loader.
+        include_unlabeled: (bool, optional). Default to False.
+            Whether to load the unlabeled records or not.
+        encoding: str, optional
+            Encoding for text files. Used only to extract the content of the text dataset contents.
+
+        Raises
+        ------
+        ValueError
+            If the record reader and record parser must be specified.
+            If the loader is not specified when materialize if True.
+        """
+        if not reader:
+            raise ValueError("The record reader must be specified.")
+        if not parser:
+            raise ValueError("The record parser must be specified.")
+        if materialize and not loader:
+            raise ValueError("The content loader must be specified.")
+
+        self.reader = reader
+        self.parser = parser
+        self.loader = loader
+        self.materialize = materialize
+        self.include_unlabeled = include_unlabeled
+        self.encoding = encoding
+
+    @classmethod
+    def from_export_file(
+        cls,
+        path: str,
+        dataset_type: str,
+        annotation_type: str,
+        dataset_source_path: str,
+        auth: Dict = None,
+        include_unlabeled: bool = False,
+        encoding: str = "utf-8",
+        materialize: bool = False,
+        format: str = None,
+        categories: List[str] = None,
+        includes_metadata=False,
+    ) -> "RecordReader":
+        """Initiates a RecordReader instance.
+
+        Parameters
+        ----------
+        path: str
+            Record file path.
+        dataset_type: str
+            Dataset type. Currently supports TEXT, IMAGE and DOCUMENT.
+        annotation_type: str
+            Annotation Type. Currently TEXT supports SINGLE_LABEL, MULTI_LABEL,
+            ENTITY_EXTRACTION. IMAGE supports SINGLE_LABEL, MULTI_LABEL and BOUNDING_BOX.
+            DOCUMENT supports SINGLE_LABEL and MULTI_LABEL.
+        dataset_source_path: str
+            Dataset source path.
+        auth: (dict, optional). Default None
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+        include_unlabeled: (bool, optional). Default to False.
+            Whether to load the unlabeled records or not.
+        encoding : (str, optional). Defaults to "utf-8".
+            Encoding for text files. Used only to extract the content of the text dataset contents.
+        materialize: (bool, optional). Defaults to False.
+            Whether to materialize the content by loader.
+        format: (str, optional). Defaults to None.
+            Output format of annotations. Can be None, "spacy" for dataset
+            Entity Extraction type or "yolo" for Object Detection type.
+            When None, it outputs List[NERItem] or List[BoundingBoxItem].
+            When "spacy", it outputs List[Tuple].
+            When "yolo", it outputs List[List[Tuple]].
+        categories: (List[str], optional). Defaults to None.
+            The list of object categories in proper order for model training.
+            Example: ['cat','dog','horse']
+        includes_metadata: (bool, optional). Defaults to False.
+            Determines whether the export file includes metadata or not.
+
+        Returns
+        -------
+        RecordReader
+            A RecordReader instance.
+        """
+        reader = ExportRecordReader(
+            path=path, auth=auth, encoding="utf-8", includes_metadata=includes_metadata
+        )
+        parser = RecordParserFactory.parser(
+            annotation_type=annotation_type,
+            dataset_source_path=dataset_source_path,
+            format=format,
+            categories=categories,
+        )
+        loader = FileLoaderFactory.loader(
+            dataset_type=dataset_type,
+            auth=auth,
+        )
+        return cls(
+            reader=reader,
+            parser=parser,
+            loader=loader,
+            materialize=materialize,
+            include_unlabeled=include_unlabeled,
+            encoding=encoding,
+        )
+
+    @classmethod
+    def from_DLS(
+        cls,
+        dataset_id: str,
+        dataset_type: str,
+        annotation_type: str,
+        dataset_source_path: str,
+        compartment_id: str = None,
+        auth: Dict = None,
+        include_unlabeled: bool = False,
+        encoding: str = "utf-8",
+        materialize: bool = False,
+        format: str = None,
+        categories: List[str] = None,
+    ) -> "RecordReader":
+        """Constructs Record Reader instance.
+
+        Parameters
+        ----------
+        dataset_id: str
+            The dataset OCID.
+        dataset_type: str
+            Dataset type. Currently supports TEXT, IMAGE and DOCUMENT.
+        annotation_type: str
+            Annotation Type. Currently TEXT supports SINGLE_LABEL, MULTI_LABEL,
+            ENTITY_EXTRACTION. IMAGE supports SINGLE_LABEL, MULTI_LABEL and BOUNDING_BOX.
+            DOCUMENT supports SINGLE_LABEL and MULTI_LABEL.
+        dataset_source_path: str
+            Dataset source path.
+        compartment_id: (str, optional). Defaults to None.
+            The compartment OCID of the dataset.
+        auth: (dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+        encoding: (str, optional). Defaults to 'utf-8'.
+            Encoding for files.
+        materialize: (bool, optional). Defaults to False.
+            Whether the content of the dataset file should be loaded or it should return the file path to the content.
+            By default the content will not be loaded.
+        format: (str, optional). Defaults to None.
+            Output format of annotations. Can be None, "spacy" for dataset
+            Entity Extraction type or "yolo" for Object Detection type.
+            When None, it outputs List[NERItem] or List[BoundingBoxItem].
+            When "spacy", it outputs List[Tuple].
+            When "yolo", it outputs List[List[Tuple]].
+        categories: (List[str], optional). Defaults to None.
+            The list of object categories in proper order for model training.
+            Example: ['cat','dog','horse']
+
+        Returns
+        -------
+        RecordReader
+            The RecordReader instance.
+        """
+
+        if compartment_id is None:
+            compartment_id = NB_SESSION_COMPARTMENT_OCID or JOB_RUN_COMPARTMENT_OCID
+
+        if not compartment_id:
+            raise ValueError("The `compartment_id` must be provided.")
+
+        reader = DLSRecordReader(
+            compartment_id=compartment_id, dataset_id=dataset_id, auth=auth
+        )
+        parser = DLSRecordParserFactory.parser(
+            annotation_type=annotation_type,
+            dataset_source_path=dataset_source_path,
+            format=format,
+            categories=categories,
+        )
+        loader = FileLoaderFactory.loader(
+            dataset_type=dataset_type,
+            auth=auth,
+        )
+        return cls(
+            reader=reader,
+            parser=parser,
+            loader=loader,
+            materialize=materialize,
+            include_unlabeled=include_unlabeled,
+            encoding=encoding,
+        )
+
+    def read(self) -> Generator[Tuple[str, Union[List, str]], Any, Any]:
+        """Reads the record.
+
+        Yields
+        ------
+        Generator[Tuple[str, Union[List, str]], Any, Any]
+            File path, content and labels in a tuple.
+        """
+        for item in self.reader.read():
+            if not item:
+                return None
+            record = self.parser.parse(item)
+            if record.annotation or self.include_unlabeled:
+                if self.materialize:
+                    record.content = self.loader.load(
+                        record.path, **{"encoding": self.encoding}
+                    )
+                yield record.to_tuple()

ads/data_labeling/record.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from dataclasses import asdict, dataclass
+from typing import Any, Tuple, Union, Dict, List
+
+from ads.data_labeling.boundingbox import BoundingBoxItem
+from ads.data_labeling.ner import NERItem
+
+
+@dataclass
+class Record:
+    """Class representing Record.
+
+    Attributes
+    ----------
+    path: str
+        File path.
+    content: Any
+        Content of the record.
+    annotation: Union[Tuple, str, List[BoundingBoxItem], List[NERItem]]
+        Annotation/label of the record.
+    """
+
+    path: str = ""
+    content: Any = None
+    annotation: Union[Tuple, str, List[BoundingBoxItem], List[NERItem]] = None
+
+    def to_dict(self) -> Dict:
+        """Convert the Record instance to a dictionary.
+
+        Returns
+        -------
+        Dict
+            Dictionary representation of the Record instance.
+        """
+        return asdict(self)
+
+    def to_tuple(
+        self,
+    ) -> Tuple[str, Any, Union[Tuple, str, List[BoundingBoxItem], List[NERItem]]]:
+        """Convert the Record instance to a tuple.
+
+        Returns
+        -------
+        Tuple
+            Tuple representation of the Record instance.
+        """
+        return (self.path, self.content, self.annotation)

ads/data_labeling/visualizer/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/