oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/data_labeling/visualizer/text_visualizer.py

@@ -1,357 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-"""
-The module that helps to visualize NER Text Dataset.
-
-Methods
--------
-    render(items: List[LabeledTextItem], options: Dict = None) -> str
-        Renders NER dataset to Html format.
-
-Examples
---------
->>> record1 = LabeledTextItem("London is the capital of the United Kingdom", [NERItem('city', 0, 6), NERItem("country", 29, 14)])
->>> record2 = LabeledTextItem("Houston area contractor seeking a Sheet Metal Superintendent.", [NERItem("city", 0, 6)])
->>> result = render(items = [record1, record2], options={"default_color":"#DDEECC", "colors": {"city":"#DDEECC", "country":"#FFAAAA"}})
->>> display(HTML(result))
-"""
-
-import logging
-from dataclasses import asdict, dataclass
-from string import Template
-from typing import Dict, List, Optional
-
-import pandas as pd
-from ads.data_labeling.constants import AnnotationType
-from ads.data_labeling.ner import NERItem
-from cerberus import Validator
-
-
-logger = logging.getLogger(__name__)
-
-HTML_OPTIONS_SCHEMA = {
-    "default_color": {"nullable": True, "required": False, "type": "string"},
-    "colors": {
-        "nullable": True,
-        "required": False,
-        "type": "dict",
-    },
-}
-
-DEFAULT_COLOR = "#cedddd"
-LEN_OF_SPACY_ITEM = 3
-
-
-@dataclass
-class LabeledTextItem:
-    """Data class representing NER Item.
-
-    Attributes
-    ----------
-    txt: str
-        The labeled sentence.
-    ents: List[NERItem]
-        The list of entities.
-    """
-
-    txt: str
-    ents: List[NERItem]
-
-    def _validate(self):
-        """Validates the instance.
-
-        Returns
-        -------
-        None
-            Nothing.
-
-        Raises
-        ------
-        ValueError
-            If txt is empty.
-        WrongEntityFormat
-            If the list of entities has a wrong format.
-        AssertionError
-            In case of overlapped entities.
-        """
-        if not self.txt:
-            raise ValueError(
-                "The parameter `txt` is required and must not be an empty string."
-            )
-
-        if not isinstance(self.ents, List):
-            raise ValueError(
-                "Invalid format for the entities. The entities must be a List[NERItem]."
-            )
-
-        for entity in self.ents:
-            if entity.offset + entity.length > len(self.txt):
-                raise ValueError(
-                    f"At least one of the entities (start index: {entity.length}, offset: {entity.offset}) "
-                    f"exceeds the length of the text ({len(self.txt)})."
-                )
-
-        self.ents.sort(key=lambda x: x.offset)
-
-        for i in range(len(self.ents) - 1):
-            if self.ents[i].offset + self.ents[i].length >= self.ents[i + 1].offset:
-                raise AssertionError(
-                    "The entity data contains overlapping tokens. The first token has a start index"
-                    f" of {self.ents[i].length}, and an offset of {self.ents[i].offset}. The second token has a start "
-                    f"index of {self.ents[i + 1].length}, and an offset of {self.ents[i + 1].offset}. "
-                )
-
-    def __post_init__(self):
-        self._validate()
-
-
-@dataclass
-class RenderOptions:
-    """Data class representing render options.
-
-    Attributes
-    ----------
-    default_color: str
-        The specified default color.
-    colors: Optional[dict]
-        The multiple specified colors.
-    """
-
-    default_color: str
-    colors: Optional[dict]
-
-    @staticmethod
-    def _validate(options: dict) -> None:
-        """Validate whether the options passed in fits the defined schema.
-
-        Parameters
-        ----------
-        options: dict
-            The multiple specified colors.
-
-        Returns
-        -------
-        None
-            Nothing.
-        """
-        if not options:
-            return None
-
-        validator = Validator(HTML_OPTIONS_SCHEMA)
-        valid = validator.validate(options)
-        if not valid:
-            raise ValueError(validator.errors)
-
-    @classmethod
-    def from_dict(cls, options: dict) -> "RenderOptions":
-        """Constructs an instance of RenderOptions from a dictionary.
-
-        Parameters
-        ----------
-        options: dict
-            Render options in dictionary format.
-
-        Returns
-        -------
-        RenderOptions
-            The instance of RenderOptions.
-        """
-        if not options:
-            return cls(default_color=DEFAULT_COLOR, colors={})
-
-        RenderOptions._validate(options)
-
-        return cls(
-            options.get("default_color", DEFAULT_COLOR), options.get("colors", {}) or {}
-        )
-
-    def to_dict(self):
-        """Converts RenderOptions instance to dictionary format.
-
-        Returns
-        -------
-        dict
-            The render options in dictionary format.
-        """
-        return asdict(self)
-
-    def __repr__(self) -> str:
-        return self.to_dict()
-
-
-class TextLabeledDataFormatter:
-    """The TextLabeledDataFormatter class to render NER items into Html format."""
-
-    _ITEM_TEMPLATE = (
-        '<span style="background-color: $color; padding: 5px; margin: 0px 5px; border-radius: 5px;">'
-        '<span style="margin-right: 5px;">$entity</span>'
-        '<span style="text-transform: uppercase; font-weight: bold; font-size:0.8em;">$label</span>'
-        "</span>"
-    )
-    _ROW_TEMPLATE = '<div key=$key style="margin-top:10px; line-height:2em">$row</div>'
-
-    @staticmethod
-    def render(items: List[LabeledTextItem], options: Dict = None) -> str:
-        """Renders NER dataset to Html format.
-
-        Parameters
-        ----------
-        items: List[LabeledTextItem]
-            Items to render.
-        options: Optional[dict]
-            Render options.
-
-        Returns
-        -------
-        str
-            Html representation of rendered NER dataset.
-
-        Raises
-        ------
-        ValueError
-            If items not provided.
-        TypeError
-            If items provided in a wrong format.
-        """
-        if not items:
-            raise ValueError("The parameter `items` is required.")
-
-        if not isinstance(items, list) or not all(
-            isinstance(x, LabeledTextItem) for x in items
-        ):
-            raise TypeError(
-                "Wrong format for the items. Items should be a `List[LabeledTextItem]`."
-            )
-
-        render_options = RenderOptions.from_dict(options)
-        item_template = Template(TextLabeledDataFormatter._ITEM_TEMPLATE)
-        row_template = Template(TextLabeledDataFormatter._ROW_TEMPLATE)
-        result = []
-
-        for item_index, item in enumerate(items):
-            current_index = 0
-            accum = []
-            for e in item.ents:
-                start = e.offset
-                end = e.offset + e.length
-                label = e.label
-                accum.append(item.txt[current_index:start])
-                accum.append(
-                    item_template.substitute(
-                        {
-                            "color": render_options.colors.get(
-                                label, render_options.default_color
-                            ),
-                            "entity": item.txt[start:end],
-                            "label": label,
-                        }
-                    )
-                )
-                current_index = end
-
-            accum.append(item.txt[current_index : len(item.txt)])
-            result.append(
-                row_template.substitute({"key": str(item_index), "row": "".join(accum)})
-            )
-
-        return "".join(result)
-
-
-def _df_to_ner_items(
-    df: pd.DataFrame,
-    content_column: str = "Content",
-    annotations_column: str = "Annotations",
-) -> List[LabeledTextItem]:
-    """Converts pandas dataframe into a list of LabeledTextItem.
-
-    Parameters
-    ----------
-    df: pd.DataFrame
-        The Pandas dataframe to convert.
-    content_column: Optional[str]
-            The column name with the content data.
-    annotations_column: Optional[str]
-        The column name for the annotations list.
-
-    Returns
-    -------
-    List[LabeledTextItem]
-        The list of LabeledTextItem objects.
-
-    Raises
-    ------
-    TypeError
-        If input data not a pandas dataframe.
-    ValueError
-        If input data has wrong format.
-    """
-    if not isinstance(df, pd.DataFrame):
-        raise TypeError("The parameter `df` must be a Pandas dataframe.")
-    if content_column not in list(df.columns):
-        raise ValueError(
-            f"The parameter `df` is invalid. It must have a column named `{content_column}`."
-        )
-    if annotations_column not in list(df.columns):
-        raise ValueError(
-            f"The parameter `df` is invalid. It must have a column named `{annotations_column}`."
-        )
-
-    if df[content_column].isnull().values.any():
-        logger.warning(
-            "The parameter `df` includes records where the text content is not "
-            "materialized. These records will be ignored. Use `materialize=True` "
-            "to load the content."
-        )
-
-    result = []
-    for item in df.T.to_dict().values():
-        if item[annotations_column] and not isinstance(item[annotations_column], list):
-            raise ValueError(
-                f"The parameter `df` is invalid. The column {annotations_column} "
-                "must be of type `List[NERItem]`."
-            )
-
-        if item[content_column]:
-            if (
-                isinstance(item[annotations_column][0], tuple)
-                and len(item[annotations_column][0]) == LEN_OF_SPACY_ITEM
-            ):
-                ents = [NERItem.from_spacy(ent) for ent in item[annotations_column]]
-            else:
-                ents = item[annotations_column] or []
-            result.append(
-                LabeledTextItem(
-                    txt=item[content_column],
-                    ents=ents,
-                )
-            )
-
-    return result
-
-
-def render(items: List[LabeledTextItem], options: Dict = None) -> str:
-    """Renders NER dataset to Html format.
-
-    Parameters
-    ----------
-    items: List[LabeledTextItem]
-        The list of NER items to render.
-    options: dict, optional
-        The options for rendering.
-
-    Returns
-    -------
-    str
-        Html string.
-
-    Examples
-    --------
-    >>> record = LabeledTextItem("London is the capital of the United Kingdom", [NERItem('city', 0, 6), NERItem("country", 29, 14)])
-    >>> result = render(items = [record], options={"default_color":"#DDEECC", "colors": {"city":"#DDEECC", "country":"#FFAAAA"}})
-    >>> display(HTML(result))
-    """
-    return TextLabeledDataFormatter.render(items, options)

ads/database/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/database/connection.py

@@ -1,338 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import ast
-import json
-import oci
-import os
-import pathlib
-import re
-import shutil
-
-from oci.exceptions import ServiceError
-from tempfile import NamedTemporaryFile
-from zipfile import ZipFile
-
-from oci.secrets import SecretsClient
-from oci.config import from_file
-
-from ads.common.decorator.runtime_dependency import (
-    runtime_dependency,
-    OptionalDependency,
-)
-from ads.common import utils
-from ads.common import auth as authutil
-from ads.common import oci_client as oc
-from ads.vault.vault import Vault
-
-
-class Connector:
-    def __init__(
-        self,
-        secret_id: str = None,
-        key: str = None,
-        repository_path: str = None,
-        **kwargs,
-    ):
-
-        """
-        Validate that a connection could be made for the given set of connection parameters, and contruct a Connector object provided that the
-        validation is successful.
-
-        Parameters
-        ----------
-        secret_id: str, optional
-            The ocid of the secret to retrieve from Oracle Cloud Infrastructure Vault.
-        key: str, optional
-            The key to find the database directory.
-        repository_path: str, optional
-            The local database information store, default to ~/.database unless specified otherwise.
-        kwargs: dict, optional
-            Name-value pairs that are to be added to the list of connection parameters.
-            For example, database_name="mydb", database_type="oracle", username = "root", password = "example-password".
-
-        Returns
-        -------
-        A Connector object.
-        """
-
-        prio_dict = {}
-
-        if kwargs:
-
-            command_creds = {}
-            for input_key in kwargs.keys():
-                command_creds[input_key] = kwargs.get(input_key)
-
-            # declaring priority order
-            prio_dict[1] = command_creds
-
-        # get creds content from vault if secret id is provided
-        if secret_id:
-            if not bool(re.match("^ocid[0-9]?\.vaultsecret.*", secret_id)):
-                raise ValueError(f"{secret_id} is not a valid secret id.")
-
-            auth = authutil.default_signer()
-            self.secret_client = oc.OCIClientFactory(**auth).secret
-
-            secret_bundle = self.secret_client.get_secret_bundle(secret_id)
-            secret_content = ast.literal_eval(
-                Vault._secret_to_dict(secret_bundle.data.secret_bundle_content.content)
-            )
-            # declaring priority order
-            prio_dict[2] = secret_content
-
-        # get creds content from local if key is provided
-        repository_path = _get_repository_path(repository_path=repository_path)
-        if not os.path.exists(repository_path):
-            raise ValueError(f"{repository_path} does not exist.")
-
-        if key:
-            if _not_valid_key(key=key):
-                raise ValueError(f"{key} is not a valid directory name.")
-            db_path = _get_db_path(repository_path=repository_path, key=key)
-            if not os.path.exists(db_path):
-                raise ValueError(f"{db_path} does not exist.")
-            local_content = get_repository(key=key, repository_path=repository_path)
-            # declaring priority order
-            prio_dict[3] = local_content
-
-        # Combine dictionary with priority using ** operator
-        config = {}
-        for k in reversed(sorted(prio_dict.keys())):
-            config.update(**prio_dict[k])
-        self.config = config
-
-        # check database types
-        valid_database_types = ["oracle"]
-
-        if "database_type" not in self.config:
-            raise ValueError(
-                f"The database_type needs to be specified. "
-                f"Valid database types are {valid_database_types}"
-            )
-
-        if self.config["database_type"] not in valid_database_types:
-            raise ValueError(
-                f"{self.config['database_type']} is not a valid database type. "
-                f"Valid database types are {valid_database_types}"
-            )
-
-        if self.config["database_type"] == "oracle":
-            connector = OracleConnector(self.config)
-            self.uri = connector.uri
-            self.engine = connector.engine
-
-    def connect(self):
-        return self.engine.connect()
-
-    def __enter__(self):
-        self.db_connection = self.engine.connect()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        return self
-
-    def __repr__(self):  # pragma: no cover
-        return str(self.engine.url)
-
-
-class OracleConnector:
-    @runtime_dependency(module="sqlalchemy", install_from=OptionalDependency.DATA)
-    def __init__(self, oracle_connection_config):
-        self.config = oracle_connection_config
-
-        # sanity check on valid keys before making a connection
-        valid_keys = ["username", "password", "database_name"]
-        for vk in valid_keys:
-            if vk not in self.config.keys():
-                raise ValueError(f"{vk} is a required parameter to connect.")
-
-        self.uri = _create_connection_str(config=self.config)
-        self.engine = sqlalchemy.create_engine(self.uri)
-
-
-def update_repository(
-    value: dict, key: str, replace: bool = True, repository_path: str = None
-) -> dict:
-    """
-    Saves value into local database store.
-
-    Parameters
-    ----------
-    value: dict
-        The values to store locally.
-    key: str
-        The key to find the local database directory.
-    replace: bool, default to True
-        If set to false, updates the stored value.
-    repository_path:str: str, optional
-        The local database store, default to ~/.database unless specified otherwise.
-
-    Returns
-    -------
-    A dictionary of all values in the repository for the given key.
-    """
-    if _not_valid_key(key=key):
-        raise ValueError(f"{key} is not a valid directory name.")
-
-    # make local database directory
-    repository_path = _get_repository_path(repository_path=repository_path)
-    pathlib.Path(repository_path).mkdir(parents=True, exist_ok=True)
-
-    db_path = _get_db_path(repository_path=repository_path, key=key)
-    pathlib.Path(db_path).mkdir(parents=True, exist_ok=True)
-
-    db_config_path = os.path.join(db_path, "config.json")
-
-    if not replace:
-        value = _update(value, db_config_path)
-
-    with open(db_config_path, "w") as fp:
-        json.dump(value, fp)
-
-    return value
-
-
-def _update(new_value, db_config_path):
-    # update existing key's values if found different and add new key-value pairs
-    with open(db_config_path) as f:
-        old_value = json.load(f)
-    prio_dict = {1: old_value, 2: new_value}
-    return {**prio_dict[2], **prio_dict[1]}
-
-
-def get_repository(key: str, repository_path: str = None) -> dict:
-    """
-    Get all values from local database store.
-
-    Parameters
-    ----------
-    key: str
-        The key to find the database directory.
-    repository_path: str, optional
-        The path to local database store, default to ~/.database unless specified otherwise.
-
-    Returns
-    -------
-    A dictionary of all values in the store.
-    """
-    if _not_valid_key(key=key):
-        raise ValueError(f"{key} is not a valid directory name.")
-
-    # check whether repository_path exists
-    repository_path = _get_repository_path(repository_path=repository_path)
-    if not os.path.exists(repository_path):
-        raise ValueError(f"{repository_path} does not exist.")
-    # check whether db_path exists
-    db_path = _get_db_path(repository_path=repository_path, key=key)
-    if not os.path.exists(db_path):
-        raise ValueError(f"{db_path} does not exist.")
-
-    db_config_path = os.path.join(db_path, "config.json")
-    with open(db_config_path) as f:
-        return json.load(f)
-
-
-def import_wallet(wallet_path: str, key: str, repository_path: str = None) -> None:
-    """
-    Saves wallet to local database store.
-    Unzip the wallet zip file, update sqlnet.ora and store wallet files.
-
-    Parameters
-    ----------
-    wallet_path: str
-        The local path to the downloaded wallet zip file.
-    key: str
-        The key to find the database directory.
-    repository_path: str, optional
-        The local database store, default to ~/.database unless specified otherwise.
-
-    """
-    if _not_valid_key(key=key):
-        raise ValueError(f"{key} is not a valid directory name.")
-
-    # checking paths are valid
-    repository_path = _get_repository_path(repository_path=repository_path)
-    db_path = _get_db_path(repository_path=repository_path, key=key)
-
-    if not os.path.exists(db_path):
-        raise ValueError(f"{db_path} does not exist.")
-    if not os.path.exists(wallet_path):
-        raise ValueError(f"{wallet_path} does not exist.")
-
-    # Create a ZipFile Object and load wallet zip in it
-    with ZipFile(wallet_path, "r") as zipObj:
-        # Extract all the contents of zip file in current directory
-        zipObj.extractall(
-            os.path.join(os.path.expanduser("~"), f"{repository_path}/{key}")
-        )
-
-    # Add TNS_ADMIN to the environment
-    os.environ["TNS_ADMIN"] = db_path
-
-    # Update the sqlnet.ora
-    _update_sqlnet(db_path=db_path)
-
-    # Update the config.json file so that the 'database_type' is set to oracle.
-    db_config_path = os.path.join(db_path, "config.json")
-    value = _update({"database_type": "oracle"}, db_config_path)
-    with open(db_config_path, "w") as fp:
-        json.dump(value, fp)
-
-
-def _update_sqlnet(db_path):
-    sqlnet_path = os.path.join(db_path, "sqlnet.ora")
-    sqlnet_original_path = os.path.join(db_path, "sqlnet.ora.original")
-    sqlnet_backup_path = os.path.join(db_path, "sqlnet.ora.backup")
-
-    if not os.path.exists(sqlnet_original_path):
-        shutil.copy(sqlnet_path, sqlnet_original_path)
-    if os.path.exists(sqlnet_path):
-        shutil.copy(sqlnet_path, sqlnet_backup_path)
-    sqlnet_re = re.compile(
-        '(WALLET_LOCATION\s*=.*METHOD_DATA\s*=.*DIRECTORY\s*=\s*")(.*)(".*)',
-        re.IGNORECASE,
-    )
-    tmp = NamedTemporaryFile()
-
-    with open(sqlnet_path, "rt") as sqlnet:
-        for line in sqlnet:
-            tmp.write(
-                bytearray(
-                    sqlnet_re.subn(r"\1{}\3".format(db_path), line)[0], encoding="utf-8"
-                )
-            )
-    tmp.flush()
-    shutil.copy(tmp.name, sqlnet_path)
-    tmp.close()
-
-
-def _get_repository_path(repository_path):
-    return (
-        os.path.join(os.path.expanduser("~"), ".database")
-        if repository_path is None
-        else repository_path
-    )
-
-
-def _get_db_path(repository_path, key):
-    return os.path.join(repository_path, key)
-
-
-def _not_valid_key(key):
-    return bool(re.search("[../.]", key))
-
-
-def _create_connection_str(config):
-    return (
-        "oracle+cx_oracle://"
-        + config["username"]
-        + ":"
-        + config["password"]
-        + "@"
-        + config["database_name"]
-    )

ads/dataset/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-
-import logging
-
-logger = logging.getLogger(__name__)