oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/data_labeling/visualizer/text_visualizer.py

@@ -0,0 +1,357 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+"""
+The module that helps to visualize NER Text Dataset.
+
+Methods
+-------
+    render(items: List[LabeledTextItem], options: Dict = None) -> str
+        Renders NER dataset to Html format.
+
+Examples
+--------
+>>> record1 = LabeledTextItem("London is the capital of the United Kingdom", [NERItem('city', 0, 6), NERItem("country", 29, 14)])
+>>> record2 = LabeledTextItem("Houston area contractor seeking a Sheet Metal Superintendent.", [NERItem("city", 0, 6)])
+>>> result = render(items = [record1, record2], options={"default_color":"#DDEECC", "colors": {"city":"#DDEECC", "country":"#FFAAAA"}})
+>>> display(HTML(result))
+"""
+
+import logging
+from dataclasses import asdict, dataclass
+from string import Template
+from typing import Dict, List, Optional
+
+import pandas as pd
+from ads.data_labeling.constants import AnnotationType
+from ads.data_labeling.ner import NERItem
+from cerberus import Validator
+
+
+logger = logging.getLogger(__name__)
+
+HTML_OPTIONS_SCHEMA = {
+    "default_color": {"nullable": True, "required": False, "type": "string"},
+    "colors": {
+        "nullable": True,
+        "required": False,
+        "type": "dict",
+    },
+}
+
+DEFAULT_COLOR = "#cedddd"
+LEN_OF_SPACY_ITEM = 3
+
+
+@dataclass
+class LabeledTextItem:
+    """Data class representing NER Item.
+
+    Attributes
+    ----------
+    txt: str
+        The labeled sentence.
+    ents: List[NERItem]
+        The list of entities.
+    """
+
+    txt: str
+    ents: List[NERItem]
+
+    def _validate(self):
+        """Validates the instance.
+
+        Returns
+        -------
+        None
+            Nothing.
+
+        Raises
+        ------
+        ValueError
+            If txt is empty.
+        WrongEntityFormat
+            If the list of entities has a wrong format.
+        AssertionError
+            In case of overlapped entities.
+        """
+        if not self.txt:
+            raise ValueError(
+                "The parameter `txt` is required and must not be an empty string."
+            )
+
+        if not isinstance(self.ents, List):
+            raise ValueError(
+                "Invalid format for the entities. The entities must be a List[NERItem]."
+            )
+
+        for entity in self.ents:
+            if entity.offset + entity.length > len(self.txt):
+                raise ValueError(
+                    f"At least one of the entities (start index: {entity.length}, offset: {entity.offset}) "
+                    f"exceeds the length of the text ({len(self.txt)})."
+                )
+
+        self.ents.sort(key=lambda x: x.offset)
+
+        for i in range(len(self.ents) - 1):
+            if self.ents[i].offset + self.ents[i].length >= self.ents[i + 1].offset:
+                raise AssertionError(
+                    "The entity data contains overlapping tokens. The first token has a start index"
+                    f" of {self.ents[i].length}, and an offset of {self.ents[i].offset}. The second token has a start "
+                    f"index of {self.ents[i + 1].length}, and an offset of {self.ents[i + 1].offset}. "
+                )
+
+    def __post_init__(self):
+        self._validate()
+
+
+@dataclass
+class RenderOptions:
+    """Data class representing render options.
+
+    Attributes
+    ----------
+    default_color: str
+        The specified default color.
+    colors: Optional[dict]
+        The multiple specified colors.
+    """
+
+    default_color: str
+    colors: Optional[dict]
+
+    @staticmethod
+    def _validate(options: dict) -> None:
+        """Validate whether the options passed in fits the defined schema.
+
+        Parameters
+        ----------
+        options: dict
+            The multiple specified colors.
+
+        Returns
+        -------
+        None
+            Nothing.
+        """
+        if not options:
+            return None
+
+        validator = Validator(HTML_OPTIONS_SCHEMA)
+        valid = validator.validate(options)
+        if not valid:
+            raise ValueError(validator.errors)
+
+    @classmethod
+    def from_dict(cls, options: dict) -> "RenderOptions":
+        """Constructs an instance of RenderOptions from a dictionary.
+
+        Parameters
+        ----------
+        options: dict
+            Render options in dictionary format.
+
+        Returns
+        -------
+        RenderOptions
+            The instance of RenderOptions.
+        """
+        if not options:
+            return cls(default_color=DEFAULT_COLOR, colors={})
+
+        RenderOptions._validate(options)
+
+        return cls(
+            options.get("default_color", DEFAULT_COLOR), options.get("colors", {}) or {}
+        )
+
+    def to_dict(self):
+        """Converts RenderOptions instance to dictionary format.
+
+        Returns
+        -------
+        dict
+            The render options in dictionary format.
+        """
+        return asdict(self)
+
+    def __repr__(self) -> str:
+        return self.to_dict()
+
+
+class TextLabeledDataFormatter:
+    """The TextLabeledDataFormatter class to render NER items into Html format."""
+
+    _ITEM_TEMPLATE = (
+        '<span style="background-color: $color; padding: 5px; margin: 0px 5px; border-radius: 5px;">'
+        '<span style="margin-right: 5px;">$entity</span>'
+        '<span style="text-transform: uppercase; font-weight: bold; font-size:0.8em;">$label</span>'
+        "</span>"
+    )
+    _ROW_TEMPLATE = '<div key=$key style="margin-top:10px; line-height:2em">$row</div>'
+
+    @staticmethod
+    def render(items: List[LabeledTextItem], options: Dict = None) -> str:
+        """Renders NER dataset to Html format.
+
+        Parameters
+        ----------
+        items: List[LabeledTextItem]
+            Items to render.
+        options: Optional[dict]
+            Render options.
+
+        Returns
+        -------
+        str
+            Html representation of rendered NER dataset.
+
+        Raises
+        ------
+        ValueError
+            If items not provided.
+        TypeError
+            If items provided in a wrong format.
+        """
+        if not items:
+            raise ValueError("The parameter `items` is required.")
+
+        if not isinstance(items, list) or not all(
+            isinstance(x, LabeledTextItem) for x in items
+        ):
+            raise TypeError(
+                "Wrong format for the items. Items should be a `List[LabeledTextItem]`."
+            )
+
+        render_options = RenderOptions.from_dict(options)
+        item_template = Template(TextLabeledDataFormatter._ITEM_TEMPLATE)
+        row_template = Template(TextLabeledDataFormatter._ROW_TEMPLATE)
+        result = []
+
+        for item_index, item in enumerate(items):
+            current_index = 0
+            accum = []
+            for e in item.ents:
+                start = e.offset
+                end = e.offset + e.length
+                label = e.label
+                accum.append(item.txt[current_index:start])
+                accum.append(
+                    item_template.substitute(
+                        {
+                            "color": render_options.colors.get(
+                                label, render_options.default_color
+                            ),
+                            "entity": item.txt[start:end],
+                            "label": label,
+                        }
+                    )
+                )
+                current_index = end
+
+            accum.append(item.txt[current_index : len(item.txt)])
+            result.append(
+                row_template.substitute({"key": str(item_index), "row": "".join(accum)})
+            )
+
+        return "".join(result)
+
+
+def _df_to_ner_items(
+    df: pd.DataFrame,
+    content_column: str = "Content",
+    annotations_column: str = "Annotations",
+) -> List[LabeledTextItem]:
+    """Converts pandas dataframe into a list of LabeledTextItem.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        The Pandas dataframe to convert.
+    content_column: Optional[str]
+            The column name with the content data.
+    annotations_column: Optional[str]
+        The column name for the annotations list.
+
+    Returns
+    -------
+    List[LabeledTextItem]
+        The list of LabeledTextItem objects.
+
+    Raises
+    ------
+    TypeError
+        If input data not a pandas dataframe.
+    ValueError
+        If input data has wrong format.
+    """
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError("The parameter `df` must be a Pandas dataframe.")
+    if content_column not in list(df.columns):
+        raise ValueError(
+            f"The parameter `df` is invalid. It must have a column named `{content_column}`."
+        )
+    if annotations_column not in list(df.columns):
+        raise ValueError(
+            f"The parameter `df` is invalid. It must have a column named `{annotations_column}`."
+        )
+
+    if df[content_column].isnull().values.any():
+        logger.warning(
+            "The parameter `df` includes records where the text content is not "
+            "materialized. These records will be ignored. Use `materialize=True` "
+            "to load the content."
+        )
+
+    result = []
+    for item in df.T.to_dict().values():
+        if item[annotations_column] and not isinstance(item[annotations_column], list):
+            raise ValueError(
+                f"The parameter `df` is invalid. The column {annotations_column} "
+                "must be of type `List[NERItem]`."
+            )
+
+        if item[content_column]:
+            if (
+                isinstance(item[annotations_column][0], tuple)
+                and len(item[annotations_column][0]) == LEN_OF_SPACY_ITEM
+            ):
+                ents = [NERItem.from_spacy(ent) for ent in item[annotations_column]]
+            else:
+                ents = item[annotations_column] or []
+            result.append(
+                LabeledTextItem(
+                    txt=item[content_column],
+                    ents=ents,
+                )
+            )
+
+    return result
+
+
+def render(items: List[LabeledTextItem], options: Dict = None) -> str:
+    """Renders NER dataset to Html format.
+
+    Parameters
+    ----------
+    items: List[LabeledTextItem]
+        The list of NER items to render.
+    options: dict, optional
+        The options for rendering.
+
+    Returns
+    -------
+    str
+        Html string.
+
+    Examples
+    --------
+    >>> record = LabeledTextItem("London is the capital of the United Kingdom", [NERItem('city', 0, 6), NERItem("country", 29, 14)])
+    >>> result = render(items = [record], options={"default_color":"#DDEECC", "colors": {"city":"#DDEECC", "country":"#FFAAAA"}})
+    >>> display(HTML(result))
+    """
+    return TextLabeledDataFormatter.render(items, options)

ads/database/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/database/connection.py

@@ -0,0 +1,338 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import ast
+import json
+import oci
+import os
+import pathlib
+import re
+import shutil
+
+from oci.exceptions import ServiceError
+from tempfile import NamedTemporaryFile
+from zipfile import ZipFile
+
+from oci.secrets import SecretsClient
+from oci.config import from_file
+
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+from ads.common import utils
+from ads.common import auth as authutil
+from ads.common import oci_client as oc
+from ads.vault.vault import Vault
+
+
+class Connector:
+    def __init__(
+        self,
+        secret_id: str = None,
+        key: str = None,
+        repository_path: str = None,
+        **kwargs,
+    ):
+
+        """
+        Validate that a connection could be made for the given set of connection parameters, and contruct a Connector object provided that the
+        validation is successful.
+
+        Parameters
+        ----------
+        secret_id: str, optional
+            The ocid of the secret to retrieve from Oracle Cloud Infrastructure Vault.
+        key: str, optional
+            The key to find the database directory.
+        repository_path: str, optional
+            The local database information store, default to ~/.database unless specified otherwise.
+        kwargs: dict, optional
+            Name-value pairs that are to be added to the list of connection parameters.
+            For example, database_name="mydb", database_type="oracle", username = "root", password = "example-password".
+
+        Returns
+        -------
+        A Connector object.
+        """
+
+        prio_dict = {}
+
+        if kwargs:
+
+            command_creds = {}
+            for input_key in kwargs.keys():
+                command_creds[input_key] = kwargs.get(input_key)
+
+            # declaring priority order
+            prio_dict[1] = command_creds
+
+        # get creds content from vault if secret id is provided
+        if secret_id:
+            if not bool(re.match("^ocid[0-9]?\.vaultsecret.*", secret_id)):
+                raise ValueError(f"{secret_id} is not a valid secret id.")
+
+            auth = authutil.default_signer()
+            self.secret_client = oc.OCIClientFactory(**auth).secret
+
+            secret_bundle = self.secret_client.get_secret_bundle(secret_id)
+            secret_content = ast.literal_eval(
+                Vault._secret_to_dict(secret_bundle.data.secret_bundle_content.content)
+            )
+            # declaring priority order
+            prio_dict[2] = secret_content
+
+        # get creds content from local if key is provided
+        repository_path = _get_repository_path(repository_path=repository_path)
+        if not os.path.exists(repository_path):
+            raise ValueError(f"{repository_path} does not exist.")
+
+        if key:
+            if _not_valid_key(key=key):
+                raise ValueError(f"{key} is not a valid directory name.")
+            db_path = _get_db_path(repository_path=repository_path, key=key)
+            if not os.path.exists(db_path):
+                raise ValueError(f"{db_path} does not exist.")
+            local_content = get_repository(key=key, repository_path=repository_path)
+            # declaring priority order
+            prio_dict[3] = local_content
+
+        # Combine dictionary with priority using ** operator
+        config = {}
+        for k in reversed(sorted(prio_dict.keys())):
+            config.update(**prio_dict[k])
+        self.config = config
+
+        # check database types
+        valid_database_types = ["oracle"]
+
+        if "database_type" not in self.config:
+            raise ValueError(
+                f"The database_type needs to be specified. "
+                f"Valid database types are {valid_database_types}"
+            )
+
+        if self.config["database_type"] not in valid_database_types:
+            raise ValueError(
+                f"{self.config['database_type']} is not a valid database type. "
+                f"Valid database types are {valid_database_types}"
+            )
+
+        if self.config["database_type"] == "oracle":
+            connector = OracleConnector(self.config)
+            self.uri = connector.uri
+            self.engine = connector.engine
+
+    def connect(self):
+        return self.engine.connect()
+
+    def __enter__(self):
+        self.db_connection = self.engine.connect()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return self
+
+    def __repr__(self):  # pragma: no cover
+        return str(self.engine.url)
+
+
+class OracleConnector:
+    @runtime_dependency(module="sqlalchemy", install_from=OptionalDependency.DATA)
+    def __init__(self, oracle_connection_config):
+        self.config = oracle_connection_config
+
+        # sanity check on valid keys before making a connection
+        valid_keys = ["username", "password", "database_name"]
+        for vk in valid_keys:
+            if vk not in self.config.keys():
+                raise ValueError(f"{vk} is a required parameter to connect.")
+
+        self.uri = _create_connection_str(config=self.config)
+        self.engine = sqlalchemy.create_engine(self.uri)
+
+
+def update_repository(
+    value: dict, key: str, replace: bool = True, repository_path: str = None
+) -> dict:
+    """
+    Saves value into local database store.
+
+    Parameters
+    ----------
+    value: dict
+        The values to store locally.
+    key: str
+        The key to find the local database directory.
+    replace: bool, default to True
+        If set to false, updates the stored value.
+    repository_path:str: str, optional
+        The local database store, default to ~/.database unless specified otherwise.
+
+    Returns
+    -------
+    A dictionary of all values in the repository for the given key.
+    """
+    if _not_valid_key(key=key):
+        raise ValueError(f"{key} is not a valid directory name.")
+
+    # make local database directory
+    repository_path = _get_repository_path(repository_path=repository_path)
+    pathlib.Path(repository_path).mkdir(parents=True, exist_ok=True)
+
+    db_path = _get_db_path(repository_path=repository_path, key=key)
+    pathlib.Path(db_path).mkdir(parents=True, exist_ok=True)
+
+    db_config_path = os.path.join(db_path, "config.json")
+
+    if not replace:
+        value = _update(value, db_config_path)
+
+    with open(db_config_path, "w") as fp:
+        json.dump(value, fp)
+
+    return value
+
+
+def _update(new_value, db_config_path):
+    # update existing key's values if found different and add new key-value pairs
+    with open(db_config_path) as f:
+        old_value = json.load(f)
+    prio_dict = {1: old_value, 2: new_value}
+    return {**prio_dict[2], **prio_dict[1]}
+
+
+def get_repository(key: str, repository_path: str = None) -> dict:
+    """
+    Get all values from local database store.
+
+    Parameters
+    ----------
+    key: str
+        The key to find the database directory.
+    repository_path: str, optional
+        The path to local database store, default to ~/.database unless specified otherwise.
+
+    Returns
+    -------
+    A dictionary of all values in the store.
+    """
+    if _not_valid_key(key=key):
+        raise ValueError(f"{key} is not a valid directory name.")
+
+    # check whether repository_path exists
+    repository_path = _get_repository_path(repository_path=repository_path)
+    if not os.path.exists(repository_path):
+        raise ValueError(f"{repository_path} does not exist.")
+    # check whether db_path exists
+    db_path = _get_db_path(repository_path=repository_path, key=key)
+    if not os.path.exists(db_path):
+        raise ValueError(f"{db_path} does not exist.")
+
+    db_config_path = os.path.join(db_path, "config.json")
+    with open(db_config_path) as f:
+        return json.load(f)
+
+
+def import_wallet(wallet_path: str, key: str, repository_path: str = None) -> None:
+    """
+    Saves wallet to local database store.
+    Unzip the wallet zip file, update sqlnet.ora and store wallet files.
+
+    Parameters
+    ----------
+    wallet_path: str
+        The local path to the downloaded wallet zip file.
+    key: str
+        The key to find the database directory.
+    repository_path: str, optional
+        The local database store, default to ~/.database unless specified otherwise.
+
+    """
+    if _not_valid_key(key=key):
+        raise ValueError(f"{key} is not a valid directory name.")
+
+    # checking paths are valid
+    repository_path = _get_repository_path(repository_path=repository_path)
+    db_path = _get_db_path(repository_path=repository_path, key=key)
+
+    if not os.path.exists(db_path):
+        raise ValueError(f"{db_path} does not exist.")
+    if not os.path.exists(wallet_path):
+        raise ValueError(f"{wallet_path} does not exist.")
+
+    # Create a ZipFile Object and load wallet zip in it
+    with ZipFile(wallet_path, "r") as zipObj:
+        # Extract all the contents of zip file in current directory
+        zipObj.extractall(
+            os.path.join(os.path.expanduser("~"), f"{repository_path}/{key}")
+        )
+
+    # Add TNS_ADMIN to the environment
+    os.environ["TNS_ADMIN"] = db_path
+
+    # Update the sqlnet.ora
+    _update_sqlnet(db_path=db_path)
+
+    # Update the config.json file so that the 'database_type' is set to oracle.
+    db_config_path = os.path.join(db_path, "config.json")
+    value = _update({"database_type": "oracle"}, db_config_path)
+    with open(db_config_path, "w") as fp:
+        json.dump(value, fp)
+
+
+def _update_sqlnet(db_path):
+    sqlnet_path = os.path.join(db_path, "sqlnet.ora")
+    sqlnet_original_path = os.path.join(db_path, "sqlnet.ora.original")
+    sqlnet_backup_path = os.path.join(db_path, "sqlnet.ora.backup")
+
+    if not os.path.exists(sqlnet_original_path):
+        shutil.copy(sqlnet_path, sqlnet_original_path)
+    if os.path.exists(sqlnet_path):
+        shutil.copy(sqlnet_path, sqlnet_backup_path)
+    sqlnet_re = re.compile(
+        '(WALLET_LOCATION\s*=.*METHOD_DATA\s*=.*DIRECTORY\s*=\s*")(.*)(".*)',
+        re.IGNORECASE,
+    )
+    tmp = NamedTemporaryFile()
+
+    with open(sqlnet_path, "rt") as sqlnet:
+        for line in sqlnet:
+            tmp.write(
+                bytearray(
+                    sqlnet_re.subn(r"\1{}\3".format(db_path), line)[0], encoding="utf-8"
+                )
+            )
+    tmp.flush()
+    shutil.copy(tmp.name, sqlnet_path)
+    tmp.close()
+
+
+def _get_repository_path(repository_path):
+    return (
+        os.path.join(os.path.expanduser("~"), ".database")
+        if repository_path is None
+        else repository_path
+    )
+
+
+def _get_db_path(repository_path, key):
+    return os.path.join(repository_path, key)
+
+
+def _not_valid_key(key):
+    return bool(re.search("[../.]", key))
+
+
+def _create_connection_str(config):
+    return (
+        "oracle+cx_oracle://"
+        + config["username"]
+        + ":"
+        + config["password"]
+        + "@"
+        + config["database_name"]
+    )

ads/dataset/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+import logging
+
+logger = logging.getLogger(__name__)