oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/data_labeling/reader/export_record_reader.py

@@ -1,62 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from typing import Any, Dict, Generator
-from ads.data_labeling.reader.jsonl_reader import JsonlReader
-
-
-class ExportRecordReader(JsonlReader):
-    """The ExportRecordReader class to read labeled dataset records from the export.
-
-    Methods
-    -------
-    read(self) -> Generator[Dict, Any, Any]
-        Reads labeled dataset records.
-    """
-
-    def __init__(
-        self,
-        path: str,
-        auth: Dict = None,
-        encoding="utf-8",
-        includes_metadata: bool = False,
-    ) -> "ExportRecordReader":
-        """Initiates an ExportRecordReader instance.
-
-        Parameters
-        ----------
-        path: str
-            object storage path or local path for a file.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        encoding : (str, optional). Defaults to 'utf-8'.
-            Encoding of files. Only used for "TEXT" dataset.
-        includes_metadata: (bool, optional). Defaults to False.
-            Determines whether the export file includes metadata or not.
-
-        Examples
-        --------
-        >>> from ads.data_labeling.reader.export_record_reader import ExportRecordReader
-        >>> path = "your/path/to/jsonl/file.jsonl"
-        >>> from ads.common import auth as authutil
-        >>> reader = ExportRecordReader(path=path, auth=authutil.api_keys(), encoding="utf-8")
-        >>> next(reader.read())
-        """
-        super().__init__(path=path, auth=auth, encoding=encoding)
-        self._includes_metadata = includes_metadata
-
-    def read(self) -> Generator[Dict, Any, Any]:
-        """Reads labeled dataset records.
-
-        Returns
-        -------
-        Generator[Dict, Any, Any]
-            The labeled dataset records.
-        """
-        skip = 1 if self._includes_metadata else None
-        return super().read(skip=skip)

ads/data_labeling/reader/jsonl_reader.py

@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import json
-from typing import Any, Dict, Generator
-
-import fsspec
-from ads.data_labeling.interface.reader import Reader
-from ads.common import auth as authutil
-
-
-class JsonlReader(Reader):
-    """JsonlReader class which reads the file."""
-
-    def __init__(self, path: str, auth: Dict = None, encoding="utf-8") -> "JsonlReader":
-        """Initiates a JsonlReader object.
-
-        Parameters
-        ----------
-        path: str
-            object storage path or local path for a file.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        encoding : (str, optional). Defaults to 'utf-8'.
-            Encoding of files. Only used for "TEXT" dataset.
-
-        Examples
-        --------
-        >>> from ads.data_labeling.reader.jsonl_reader import JsonlReader
-        >>> path = "your/path/to/jsonl/file.jsonl"
-        >>> from ads.common import auth as authutil
-        >>> reader = JsonlReader(path=path, auth=authutil.api_keys(), encoding="utf-8")
-        >>> next(reader.read())
-        """
-        self.path = path
-        self.auth = auth or authutil.default_signer()
-        self.encoding = encoding
-
-    def read(self, skip: int = None) -> Generator[Dict, Any, Any]:
-        """Reads and yields the content of the file.
-
-        Parameters
-        ----------
-        skip: (int, optional). Defaults to None.
-            The number of records that should be skipped.
-
-        Returns
-        -------
-        Generator[Dict, Any, Any]
-            The content of the file.
-
-        Raises
-        ------
-        ValueError
-            If `skip` not empty and not a positive integer.
-        FileNotFoundError
-            When file not found.
-        """
-        if skip and (not isinstance(skip, int) or skip < 1):
-            raise ValueError("The parameter `skip` must be a positive integer.")
-        try:
-            line_number = 0
-            with fsspec.open(self.path, "r", encoding=self.encoding, **self.auth) as f:
-                for line in f:
-                    line_number += 1
-                    if skip and line_number <= skip:
-                        continue
-                    yield json.loads(line)
-        except FileNotFoundError:
-            raise FileNotFoundError(f"Path ({self.path}) not found.")

ads/data_labeling/reader/metadata_reader.py

@@ -1,203 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from typing import Dict
-
-from ads.common import auth as auth_util
-from ads.common import oci_client
-from ads.config import JOB_RUN_COMPARTMENT_OCID, NB_SESSION_COMPARTMENT_OCID
-from ads.data_labeling.interface.reader import Reader
-from ads.data_labeling.metadata import Metadata
-from ads.data_labeling.parser.export_metadata_parser import MetadataParser
-from ads.data_labeling.reader.jsonl_reader import JsonlReader
-from oci.exceptions import ServiceError
-
-
-class EmptyMetadata(Exception):   # pragma: no cover
-    """Empty Metadata."""
-
-    pass
-
-
-class ReadDatasetError(Exception):   # pragma: no cover
-    def __init__(self, id: str):
-        super().__init__(f"Error occurred in attempt to read dataset {id}.")
-
-
-class DatasetNotFoundError(Exception):   # pragma: no cover
-    def __init__(self, id: str):
-        super().__init__(f"{id} not found.")
-
-
-class MetadataReader:
-    """MetadataReader class which reads and extracts the labeled dataset metadata.
-
-    Examples
-    --------
-    >>> from ads.data_labeling import MetadataReader
-    >>> import oci
-    >>> import os
-    >>> from ads.common import auth as authutil
-    >>> reader = MetadataReader.from_export_file("metadata_export_file_path",
-    ...                                 auth=authutil.api_keys())
-    >>> reader.read()
-    """
-
-    def __init__(self, reader: Reader):
-        """Initiate a MetadataReader instance.
-
-        Parameters
-        ----------
-        reader: Reader
-            Reader instance which reads and extracts the labeled dataset metadata.
-        """
-        self._reader = reader
-
-    @classmethod
-    def from_export_file(cls, path: str, auth: Dict = None) -> "MetadataReader":
-        """Contructs a MetadataReader instance.
-
-        Parameters
-        ----------
-        path: str
-            metadata file path, can be either local or object storage path.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-
-        Returns
-        -------
-        MetadataReader
-            The MetadataReader instance whose reader is a ExportMetadataReader instance.
-        """
-        return cls(ExportMetadataReader(path=path, auth=auth))
-
-    @classmethod
-    def from_DLS(
-        cls, dataset_id: str, compartment_id: str = None, auth: dict = None
-    ) -> "MetadataReader":
-        """Contructs a MetadataReader instance.
-
-        Parameters
-        ----------
-        dataset_id: str
-            The dataset OCID.
-        compartment_id: (str, optional). Default None
-            The compartment OCID of the dataset.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-
-        Returns
-        -------
-        MetadataReader
-            The MetadataReader instance whose reader is a DLSMetadataReader instance.
-        """
-        if compartment_id is None:
-            compartment_id = NB_SESSION_COMPARTMENT_OCID or JOB_RUN_COMPARTMENT_OCID
-
-        if not compartment_id:
-            raise ValueError("The `compartment_id` must be provided.")
-
-        auth = auth or auth_util.default_signer()
-
-        return cls(
-            DLSMetadataReader(
-                dataset_id=dataset_id, compartment_id=compartment_id, auth=auth
-            )
-        )
-
-    def read(self) -> Metadata:
-        """Reads the content from the metadata file.
-
-        Returns
-        -------
-        Metadata
-            The metadata of the labeled dataset.
-        """
-        return self._reader.read()
-
-
-class DLSMetadataReader(Reader):
-    """DLSMetadataReader class which reads the metadata jsonl file from the cloud."""
-
-    def __init__(self, dataset_id: str, compartment_id: str, auth: dict):
-        """Initializes the DLS metadata reader instance.
-
-        Parameters
-        ----------
-        dataset_id: str
-            The dataset OCID.
-        compartment_id: str
-            The compartment OCID of the dataset.
-        auth: dict
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-
-        Raises
-        ------
-            ValueError: When dataset_id is empty or not a string.
-            TypeError: When dataset_id not a string.
-        """
-        if not dataset_id:
-            raise ValueError("The dataset OCID must be specified.")
-
-        if not isinstance(dataset_id, str):
-            raise TypeError("The dataset_id must be a string.")
-        self.dataset_id = dataset_id
-        self.compartment_id = compartment_id
-
-        self.dls_dp_client = oci_client.OCIClientFactory(**auth).data_labeling_dp
-
-    def read(self) -> Metadata:
-        """Reads the content from the metadata file.
-
-        Returns
-        -------
-        Metadata
-            The metadata of the labeled dataset.
-
-        Raises
-        ------
-        DatasetNotFoundError
-            If dataset not found.
-        ReadDatasetError
-            If any error occured in attempt to read dataset.
-        """
-        try:
-            dataset_response = self.dls_dp_client.get_dataset(self.dataset_id)
-        except ServiceError as service_err:
-            if service_err.status == 404:
-                raise DatasetNotFoundError(self.dataset_id)
-            raise ReadDatasetError(self.dataset_id)
-        return Metadata.from_dls_dataset(dataset_response.data)
-
-
-class ExportMetadataReader(JsonlReader):
-    """ExportMetadataReader class which reads the metadata jsonl file from local/object
-    storage path."""
-
-    def read(self) -> Metadata:
-        """Reads the content from the metadata file.
-
-        Returns
-        -------
-        Metadata
-            The metadata of the labeled dataset.
-        """
-
-        try:
-            return MetadataParser.parse(next(super().read()))
-        except StopIteration:
-            raise EmptyMetadata(
-                "The dataset metadata file is invalid. It appears to be empty. "
-                "Use the `DataLabeling.export()` method to create a new dataset metadata file."
-            )
-        except Exception as e:
-            raise e

ads/data_labeling/reader/record_reader.py

@@ -1,263 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from typing import Any, Dict, Generator, List, Tuple, Union
-
-from ads.config import JOB_RUN_COMPARTMENT_OCID, NB_SESSION_COMPARTMENT_OCID
-from ads.data_labeling.interface.loader import Loader
-from ads.data_labeling.interface.parser import Parser
-from ads.data_labeling.interface.reader import Reader
-from ads.data_labeling.loader.file_loader import FileLoaderFactory
-from ads.data_labeling.parser.dls_record_parser import DLSRecordParserFactory
-from ads.data_labeling.parser.export_record_parser import RecordParserFactory
-from ads.data_labeling.reader.dls_record_reader import DLSRecordReader
-from ads.data_labeling.reader.export_record_reader import ExportRecordReader
-
-
-class RecordReader:
-    """Record Reader Class consists of parser, reader and loader. Reader reads the
-    the content from the record file. Parser parses the label for each record. And
-    Loader loads the content of the file path in that record.
-
-    Examples
-    --------
-    >>> import os
-    >>> import oci
-    >>> from ads.data_labeling import RecordReader
-    >>> from ads.common import auth as authutil
-    >>> file_path = "/path/to/your_record.jsonl"
-    >>> dataset_type = "IMAGE"
-    >>> annotation_type = "BOUNDING_BOX"
-    >>> record_reader = RecordReader.from_export_file(file_path, dataset_type, annotation_type, "image_file_path", authutil.api_keys())
-    >>> next(record_reader.read())
-    """
-
-    def __init__(
-        self,
-        reader: Reader,
-        parser: Parser,
-        loader: Loader = None,
-        include_unlabeled: bool = False,
-        encoding: str = "utf-8",
-        materialize: bool = False,
-    ) -> "RecordReader":
-        """Initiates a RecordReader instance.
-
-        Parameters
-        ----------
-        reader: Reader
-            Reader instance to read content from the record file.
-        parser: Parser
-            Parser instance to parse the labels from record file.
-        loader: Loader. Defaults to None.
-            Loader instance to load the content from the file path in the record.
-        materialize: bool, optional. Defaults to False.
-            Whether to materialize the content using loader.
-        include_unlabeled: (bool, optional). Default to False.
-            Whether to load the unlabeled records or not.
-        encoding: str, optional
-            Encoding for text files. Used only to extract the content of the text dataset contents.
-
-        Raises
-        ------
-        ValueError
-            If the record reader and record parser must be specified.
-            If the loader is not specified when materialize if True.
-        """
-        if not reader:
-            raise ValueError("The record reader must be specified.")
-        if not parser:
-            raise ValueError("The record parser must be specified.")
-        if materialize and not loader:
-            raise ValueError("The content loader must be specified.")
-
-        self.reader = reader
-        self.parser = parser
-        self.loader = loader
-        self.materialize = materialize
-        self.include_unlabeled = include_unlabeled
-        self.encoding = encoding
-
-    @classmethod
-    def from_export_file(
-        cls,
-        path: str,
-        dataset_type: str,
-        annotation_type: str,
-        dataset_source_path: str,
-        auth: Dict = None,
-        include_unlabeled: bool = False,
-        encoding: str = "utf-8",
-        materialize: bool = False,
-        format: str = None,
-        categories: List[str] = None,
-        includes_metadata=False,
-    ) -> "RecordReader":
-        """Initiates a RecordReader instance.
-
-        Parameters
-        ----------
-        path: str
-            Record file path.
-        dataset_type: str
-            Dataset type. Currently supports TEXT, IMAGE and DOCUMENT.
-        annotation_type: str
-            Annotation Type. Currently TEXT supports SINGLE_LABEL, MULTI_LABEL,
-            ENTITY_EXTRACTION. IMAGE supports SINGLE_LABEL, MULTI_LABEL and BOUNDING_BOX.
-            DOCUMENT supports SINGLE_LABEL and MULTI_LABEL.
-        dataset_source_path: str
-            Dataset source path.
-        auth: (dict, optional). Default None
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        include_unlabeled: (bool, optional). Default to False.
-            Whether to load the unlabeled records or not.
-        encoding : (str, optional). Defaults to "utf-8".
-            Encoding for text files. Used only to extract the content of the text dataset contents.
-        materialize: (bool, optional). Defaults to False.
-            Whether to materialize the content by loader.
-        format: (str, optional). Defaults to None.
-            Output format of annotations. Can be None, "spacy" for dataset
-            Entity Extraction type or "yolo" for Object Detection type.
-            When None, it outputs List[NERItem] or List[BoundingBoxItem].
-            When "spacy", it outputs List[Tuple].
-            When "yolo", it outputs List[List[Tuple]].
-        categories: (List[str], optional). Defaults to None.
-            The list of object categories in proper order for model training.
-            Example: ['cat','dog','horse']
-        includes_metadata: (bool, optional). Defaults to False.
-            Determines whether the export file includes metadata or not.
-
-        Returns
-        -------
-        RecordReader
-            A RecordReader instance.
-        """
-        reader = ExportRecordReader(
-            path=path, auth=auth, encoding="utf-8", includes_metadata=includes_metadata
-        )
-        parser = RecordParserFactory.parser(
-            annotation_type=annotation_type,
-            dataset_source_path=dataset_source_path,
-            format=format,
-            categories=categories,
-        )
-        loader = FileLoaderFactory.loader(
-            dataset_type=dataset_type,
-            auth=auth,
-        )
-        return cls(
-            reader=reader,
-            parser=parser,
-            loader=loader,
-            materialize=materialize,
-            include_unlabeled=include_unlabeled,
-            encoding=encoding,
-        )
-
-    @classmethod
-    def from_DLS(
-        cls,
-        dataset_id: str,
-        dataset_type: str,
-        annotation_type: str,
-        dataset_source_path: str,
-        compartment_id: str = None,
-        auth: Dict = None,
-        include_unlabeled: bool = False,
-        encoding: str = "utf-8",
-        materialize: bool = False,
-        format: str = None,
-        categories: List[str] = None,
-    ) -> "RecordReader":
-        """Constructs Record Reader instance.
-
-        Parameters
-        ----------
-        dataset_id: str
-            The dataset OCID.
-        dataset_type: str
-            Dataset type. Currently supports TEXT, IMAGE and DOCUMENT.
-        annotation_type: str
-            Annotation Type. Currently TEXT supports SINGLE_LABEL, MULTI_LABEL,
-            ENTITY_EXTRACTION. IMAGE supports SINGLE_LABEL, MULTI_LABEL and BOUNDING_BOX.
-            DOCUMENT supports SINGLE_LABEL and MULTI_LABEL.
-        dataset_source_path: str
-            Dataset source path.
-        compartment_id: (str, optional). Defaults to None.
-            The compartment OCID of the dataset.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        encoding: (str, optional). Defaults to 'utf-8'.
-            Encoding for files.
-        materialize: (bool, optional). Defaults to False.
-            Whether the content of the dataset file should be loaded or it should return the file path to the content.
-            By default the content will not be loaded.
-        format: (str, optional). Defaults to None.
-            Output format of annotations. Can be None, "spacy" for dataset
-            Entity Extraction type or "yolo" for Object Detection type.
-            When None, it outputs List[NERItem] or List[BoundingBoxItem].
-            When "spacy", it outputs List[Tuple].
-            When "yolo", it outputs List[List[Tuple]].
-        categories: (List[str], optional). Defaults to None.
-            The list of object categories in proper order for model training.
-            Example: ['cat','dog','horse']
-
-        Returns
-        -------
-        RecordReader
-            The RecordReader instance.
-        """
-
-        if compartment_id is None:
-            compartment_id = NB_SESSION_COMPARTMENT_OCID or JOB_RUN_COMPARTMENT_OCID
-
-        if not compartment_id:
-            raise ValueError("The `compartment_id` must be provided.")
-
-        reader = DLSRecordReader(
-            compartment_id=compartment_id, dataset_id=dataset_id, auth=auth
-        )
-        parser = DLSRecordParserFactory.parser(
-            annotation_type=annotation_type,
-            dataset_source_path=dataset_source_path,
-            format=format,
-            categories=categories,
-        )
-        loader = FileLoaderFactory.loader(
-            dataset_type=dataset_type,
-            auth=auth,
-        )
-        return cls(
-            reader=reader,
-            parser=parser,
-            loader=loader,
-            materialize=materialize,
-            include_unlabeled=include_unlabeled,
-            encoding=encoding,
-        )
-
-    def read(self) -> Generator[Tuple[str, Union[List, str]], Any, Any]:
-        """Reads the record.
-
-        Yields
-        ------
-        Generator[Tuple[str, Union[List, str]], Any, Any]
-            File path, content and labels in a tuple.
-        """
-        for item in self.reader.read():
-            if not item:
-                return None
-            record = self.parser.parse(item)
-            if record.annotation or self.include_unlabeled:
-                if self.materialize:
-                    record.content = self.loader.load(
-                        record.path, **{"encoding": self.encoding}
-                    )
-                yield record.to_tuple()

ads/data_labeling/record.py

@@ -1,52 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from dataclasses import asdict, dataclass
-from typing import Any, Tuple, Union, Dict, List
-
-from ads.data_labeling.boundingbox import BoundingBoxItem
-from ads.data_labeling.ner import NERItem
-
-
-@dataclass
-class Record:
-    """Class representing Record.
-
-    Attributes
-    ----------
-    path: str
-        File path.
-    content: Any
-        Content of the record.
-    annotation: Union[Tuple, str, List[BoundingBoxItem], List[NERItem]]
-        Annotation/label of the record.
-    """
-
-    path: str = ""
-    content: Any = None
-    annotation: Union[Tuple, str, List[BoundingBoxItem], List[NERItem]] = None
-
-    def to_dict(self) -> Dict:
-        """Convert the Record instance to a dictionary.
-
-        Returns
-        -------
-        Dict
-            Dictionary representation of the Record instance.
-        """
-        return asdict(self)
-
-    def to_tuple(
-        self,
-    ) -> Tuple[str, Any, Union[Tuple, str, List[BoundingBoxItem], List[NERItem]]]:
-        """Convert the Record instance to a tuple.
-
-        Returns
-        -------
-        Tuple
-            Tuple representation of the Record instance.
-        """
-        return (self.path, self.content, self.annotation)

ads/data_labeling/visualizer/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/