oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/data_labeling/parser/dls_record_parser.py

@@ -1,388 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from abc import abstractmethod
-from typing import List, Union
-
-from ads.common import auth as authutil
-from ads.common import oci_client
-from ads.data_labeling.boundingbox import BoundingBoxItem
-from ads.data_labeling.constants import AnnotationType, Formats
-from ads.data_labeling.interface.parser import Parser
-from ads.data_labeling.ner import NERItem
-from ads.data_labeling.reader.dls_record_reader import OCIRecordSummary
-from ads.data_labeling.record import Record
-from oci.data_labeling_service_dataplane.models import (
-    AnnotationSummary as OCIAnnotationSummary,
-)
-from oci.data_labeling_service_dataplane.models.annotation import (
-    Annotation as OCIAnnotation,
-)
-from oci.exceptions import ServiceError
-
-
-class ReadAnnotationError(Exception):   # pragma: no cover
-    def __init__(self, id: str):
-        super().__init__(f"Error occurred in attempt to read annotation {id}.")
-
-
-class AnnotationNotFoundError(Exception):   # pragma: no cover
-    def __init__(self, id: str):
-        super().__init__(f"{id} not found.")
-
-
-class DLSRecordParser(Parser):   # pragma: no cover
-    """DLSRecordParser class which parses the labels from the record."""
-
-    def __init__(
-        self,
-        dataset_source_path: str,
-        auth: dict = None,
-        format: str = None,
-        categories: List[str] = None,
-    ) -> "DLSRecordParser":
-        """Initiates a DLSRecordParser instance.
-
-        Parameters
-        ----------
-        dataset_source_path: str
-            Dataset source path.
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-        format: (str, optional). Defaults to None.
-            Output format of annotations.
-        categories: (List[str], optional). Defaults to None.
-            The list of object categories in proper order for model training.
-            Example: ['cat','dog','horse']
-
-        Returns
-        -------
-        DLSRecordParser
-            DLSRecordParser instance.
-        """
-        auth = auth or authutil.default_signer()
-        self.dataset_source_path = dataset_source_path
-        self.format = format
-        self.categories = categories
-        self.dls_dp_client = oci_client.OCIClientFactory(**auth).data_labeling_dp
-
-    def _get_annotation_details(
-        self, oci_annotation_summary: List[OCIAnnotationSummary]
-    ) -> List[OCIAnnotation]:
-        """Get list of annotation details.
-
-        Parameters
-        ----------
-        oci_annotation_summary: List[OCIAnnotationSummary]
-            The list of OCIAnnotationSummary objects.
-
-        Returns
-        -------
-        List[OCIAnnotation]
-            List of OCI annotations.
-        """
-        result = []
-        if oci_annotation_summary:
-            for annotation in oci_annotation_summary:
-                try:
-                    annotation_details = self.dls_dp_client.get_annotation(
-                        annotation.id
-                    ).data
-                except ServiceError as service_err:
-                    if service_err.status == 404:
-                        raise AnnotationNotFoundError(annotation.id)
-                    raise ReadAnnotationError(annotation.id)
-                result.append(annotation_details)
-        return result
-
-    def parse(self, oci_record_summary: OCIRecordSummary) -> Record:
-        """Extracts the annotations.
-        Constructs and returns a Record instance which contains the file path and the labels.
-
-        Parameters
-        ----------
-        oci_record_summary: OCIRecordSummary
-            The summary information about the record.
-
-        Returns
-        -------
-        Record
-            Record instance which contains the file path and the annotations.
-        """
-        return Record(
-            path=self.dataset_source_path + oci_record_summary.record.name,
-            annotation=self._extract_annotations(
-                self._get_annotation_details(oci_record_summary.annotation),
-                format=self.format,
-                categories=self.categories,
-            ),
-        )
-
-    @abstractmethod
-    def _extract_annotations(
-        self, oci_annotation: List[OCIAnnotation], **kwargs
-    ) -> Union[str, List[str], List[BoundingBoxItem], List[NERItem]]:
-        """Extracts annotations from the record content. Each Parser class
-        needs to implement this function.
-
-        Parameters
-        ----------
-        oci_annotation: List[OCIAnnotation]
-            List of OCI annotations.
-        kwargs: Dict
-            format: str
-                Output format of annotations. Can be "spacy" or "yolo".
-
-        Returns
-        -------
-        Union[str, List[str], List[BoundingBoxItem], List[NERItem]]
-            Label(s).
-        """
-        pass
-
-
-class DLSSingleLabelRecordParser(DLSRecordParser):
-    """SingleLabelRecordParser class which parses the label of Single label data."""
-
-    def _extract_annotations(
-        self, oci_annotation: List[OCIAnnotation], **kwargs
-    ) -> Union[str, None]:
-        """Extract the labels of the single label annotation class.
-
-        Parameters
-        ----------
-        oci_annotation: List[OCIAnnotation]
-            List of OCI annotations.
-
-        Returns
-        -------
-        Union[str, None]
-            A label or None for the unlabeled record.
-        """
-        if oci_annotation:
-            return oci_annotation[0].entities[0].labels[0].label
-        else:
-            return None
-
-
-class DLSMultiLabelRecordParser(DLSRecordParser):
-    """MultiLabelRecordParser class which parses the label of Multiple label data."""
-
-    def _extract_annotations(
-        self, oci_annotation: List[OCIAnnotation], **kwargs
-    ) -> Union[List[str], None]:
-        """Extract labels of the Multi label annotation class.
-
-        Parameters
-        ----------
-        oci_annotation: List[OCIAnnotation]
-            List of OCI annotations.
-
-        Returns
-        -------
-        Union[List[str], None]
-            List of labels or None for the unlabeled record.
-        """
-        if oci_annotation:
-            return [
-                annotation_item.label
-                for annotation_item in oci_annotation[0].entities[0].labels
-            ]
-        else:
-            return None
-
-
-class DLSNERRecordParser(DLSRecordParser):
-    """NERRecordParser class which parses the label of NER label data."""
-
-    def _extract_annotations(
-        self, oci_annotation: List[OCIAnnotation], **kwargs
-    ) -> Union[List[NERItem], None]:
-        """Extracts the labels of the NER annotation class.
-
-        Parameters
-        ----------
-        oci_annotation: List[OCIAnnotation]
-            List of OCI annotations.
-        kwargs: Dict
-            format: str
-                Output format of annotations. Can be "spacy" or None.
-                When None, it outputs List[NERItem]. When "spacy", it
-                outputs List[Tuple].
-
-        Returns
-        -------
-        Union[List[NERItem], None]
-            The list of NERItem objects.
-        """
-        items = []
-        format = kwargs.get("format", None)
-        if oci_annotation:
-            for e in oci_annotation[0].entities:
-                label = e.labels[0].label
-                offset = e.text_span.offset
-                length = e.text_span.length
-                item = NERItem(label=label, offset=int(offset), length=int(length))
-                if (
-                    format
-                    and isinstance(format, str)
-                    and format.lower() == Formats.SPACY
-                ):
-                    item = item.to_spacy()
-                items.append(item)
-        return items
-
-
-class DLSBoundingBoxRecordParser(DLSRecordParser):
-    """BoundingBoxRecordParser class which parses the label of BoundingBox label data."""
-
-    def _extract_annotations(
-        self, oci_annotation: List[OCIAnnotation], **kwargs
-    ) -> Union[List[BoundingBoxItem], None]:
-        """Extracts the labels of the Object Detection annotation class.
-
-        Parameters
-        ----------
-        oci_annotation: List[OCIAnnotation]
-            List of OCI annotations.
-        kwargs: Dict
-            format: str
-                Output format of annotations. Can be None or "yolo".
-                When None, it outputs List[BoundingBoxItem]. When "yolo", it
-                outputs List[List[Tuple]].
-            categories: (List[str], optional).
-                 The list of object categories in proper order for model training.
-                 Only used when bounding box annotations are in YOLO format.
-
-        Returns
-        -------
-        Union[List[BoundingBoxItem], None]
-            The list of BoundingBoxItem objects.
-        """
-        format = kwargs.get("format", None)
-        categories = kwargs.get("categories", None)
-
-        items = []
-        if oci_annotation:
-            for e in oci_annotation[0].entities:
-                labels = [l.label for l in e.labels]
-                coords = e.bounding_polygon.normalized_vertices
-                top_left = (float(coords[0].x), float(coords[0].y))
-                bottom_left = (float(coords[1].x), float(coords[1].y))
-                bottom_right = (float(coords[2].x), float(coords[2].y))
-                top_right = (float(coords[3].x), float(coords[3].y))
-                item = BoundingBoxItem(
-                    labels=labels,
-                    bottom_left=bottom_left,
-                    top_left=top_left,
-                    top_right=top_right,
-                    bottom_right=bottom_right,
-                )
-                if (
-                    format
-                    and isinstance(format, str)
-                    and format.lower() == Formats.YOLO
-                ):
-                    item = item.to_yolo(categories=categories)
-                items.append(item)
-
-        return items
-
-
-class DLSRecordParserFactory:
-    """DLSRecordParserFactory class which contains a list of registered parsers
-    and allows to register new DLSRecordParsers.
-
-    Current parsers include:
-        * DLSSingleLabelRecordParser
-        * DLSMultiLabelRecordParser
-        * DLSNERRecordParser
-        * DLSBoundingBoxRecordParser
-    """
-
-    _parsers = {
-        AnnotationType.SINGLE_LABEL: DLSSingleLabelRecordParser,
-        AnnotationType.MULTI_LABEL: DLSMultiLabelRecordParser,
-        AnnotationType.ENTITY_EXTRACTION: DLSNERRecordParser,
-        AnnotationType.BOUNDING_BOX: DLSBoundingBoxRecordParser,
-    }
-
-    @staticmethod
-    def parser(
-        annotation_type: str,
-        dataset_source_path: str,
-        format: str = None,
-        categories: List[str] = None,
-        auth: dict = None,
-    ) -> "DLSRecordParser":
-        """Gets the parser based on the annotation_type.
-
-        Parameters
-        ----------
-        annotation_type: str
-            Annotation type which can be SINGLE_LABEL, MULTI_LABEL, ENTITY_EXTRACTION
-            and BOUNDING_BOX.
-        dataset_source_path: str
-            Dataset source path.
-        format: (str, optional). Defaults to None.
-            Output format of annotations. Can be None, "spacy" for dataset
-            Entity Extraction type or "yolo" for Object Detection type.
-            When None, it outputs List[NERItem] or List[BoundingBoxItem].
-            When "spacy", it outputs List[Tuple].
-            When "yolo", it outputs List[List[Tuple]].
-        categories: (List[str], optional). Defaults to None.
-            The list of object categories in proper order for model training.
-            Example: ['cat','dog','horse']
-        auth: (dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
-            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
-            authentication signer and kwargs required to instantiate IdentityClient object.
-
-        Returns
-        -------
-        DLSRecordParser
-            DLSRecordParser corresponding to the annotation type.
-
-        Raises
-        ------
-        ValueError
-            If annotation_type is not supported.
-        """
-
-        if not annotation_type in DLSRecordParserFactory._parsers:
-            raise ValueError(
-                f"The {annotation_type} is not supported. Choose from "
-                f"`{AnnotationType.SINGLE_LABEL}`, `{AnnotationType.MULTI_LABEL}`, "
-                f"`{AnnotationType.ENTITY_EXTRACTION}` and `{AnnotationType.BOUNDING_BOX}`."
-            )
-
-        return DLSRecordParserFactory._parsers[annotation_type](
-            dataset_source_path=dataset_source_path,
-            format=format,
-            categories=categories,
-            auth=auth or authutil.default_signer(),
-        )
-
-    @classmethod
-    def register(cls, annotation_type: str, parser: DLSRecordParser) -> None:
-        """Registers a new parser.
-
-        Parameters
-        ----------
-        annotation_type: str
-            Annotation type which can be SINGLE_LABEL, MULTI_LABEL, ENTITY_EXTRACTION
-            and BOUNDING_BOX.
-        parser: DLSRecordParser
-            A new Parser class to be registered.
-
-        Returns
-        -------
-        None
-            Nothing.
-        """
-        cls._parsers[annotation_type] = parser

ads/data_labeling/parser/export_metadata_parser.py

@@ -1,94 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from typing import Any, Dict
-
-from ads.common.object_storage_details import ObjectStorageDetails
-from ads.data_labeling.interface.parser import Parser
-from ads.data_labeling.metadata import Metadata
-
-
-class MetadataParser(Parser):
-    """MetadataParser class which parses the metadata from the record."""
-
-    EXPECTED_KEYS = [
-        "id",
-        "compartmentId",
-        "displayName",
-        "labelsSet",
-        "annotationFormat",
-        "datasetSourceDetails",
-        "datasetFormatDetails",
-    ]
-
-    @staticmethod
-    def parse(json_data: Dict[Any, Any]) -> "Metadata":
-        """Parses the metadata jsonl file.
-
-        Parameters
-        ----------
-        json_data: dict
-            dictionary format of the metadata jsonl file content.
-
-        Returns
-        -------
-        Metadata
-            Metadata object which contains the useful fields from the metadata jsonl file
-        """
-        MetadataParser._validate(json_data)
-        source_path = ObjectStorageDetails(
-            json_data["datasetSourceDetails"]["bucket"],
-            json_data["datasetSourceDetails"]["namespace"],
-            json_data["datasetSourceDetails"]["prefix"],
-        ).path
-
-        records_path = ""
-        if "recordFiles" in json_data:
-            records_path = ObjectStorageDetails(
-                json_data["recordFiles"][0]["bucket"],
-                json_data["recordFiles"][0]["namespace"],
-                json_data["recordFiles"][0]["path"],
-            ).path
-
-        return Metadata(
-            source_path=source_path,
-            records_path=records_path,
-            labels=[clss["name"] for clss in json_data["labelsSet"]],
-            dataset_name=json_data["displayName"],
-            compartment_id=json_data["compartmentId"],
-            dataset_id=json_data["id"],
-            annotation_type=json_data["annotationFormat"],
-            dataset_type=json_data["datasetFormatDetails"]["formatType"],
-        )
-
-    @staticmethod
-    def _validate(json_data: Dict[Any, Any]) -> None:
-        """Validates the metadata jsonl file to ensure it contains certain fields.
-
-        Parameters
-        ----------
-        json_data: dict
-            dictionary format of the metadata jsonl file content.
-        """
-
-        def invalid_message(param):
-            return (
-                f"The dataset metadata file is invalid. The field '{param}' is required but it is missing. "
-                + "Update the metadata file or use the `DataLabeling.export()` method "
-                + "to create a new dataset metadata file."
-            )
-
-        for k in MetadataParser.EXPECTED_KEYS:
-            if k not in json_data:
-                raise ValueError(f"{invalid_message(k)}")
-        expected_list_format = ["labelsSet", "recordFiles"]
-        for k in expected_list_format:
-            if k in json_data and not isinstance(json_data[k], list):
-                raise ValueError(f"{invalid_message(k)}")
-        expected_dict_format = ["datasetSourceDetails", "datasetFormatDetails"]
-        for k in expected_dict_format:
-            if not isinstance(json_data[k], dict):
-                raise ValueError(f"{invalid_message(k)}")