oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/aqua/dummy_data/readme.md

@@ -0,0 +1,26 @@
+# Model Card: Dummy Text Generator
+## Description
+This is a simple dummy text generator model developed using Hugging Face's Transformers library. It generates random text based on a pre-trained language model.
+## Model Details
+- Model Name: DummyTextGenerator
+- Model Architecture: GPT-2
+- Model Size: 125M parameters
+- Training Data: Random text from the internet
+## Usage
+You can use this model to generate dummy text for various purposes, such as testing text processing pipelines or generating placeholder text for design mockups.
+Here's an example of how to use it in Python:
+```python
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+model_name = "dummy-text-generator"
+tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+model = GPT2LMHeadModel.from_pretrained(model_name)
+prompt = "Once upon a time"
+input_ids = tokenizer.encode(prompt, return_tensors="pt")
+output = model.generate(input_ids, max_length=50, num_return_sequences=1, pad_token_id=tokenizer.eos_token_id)
+generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
+print(generated_text)
+```
+## Evaluation
+The model does not perform any meaningful text generation but can be used for basic testing purposes.
+## License
+This model is released under the MIT License.

ads/aqua/evaluation/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+from ads.aqua.evaluation.evaluation import AquaEvaluationApp
+
+__all__ = ["AquaEvaluationApp"]

ads/aqua/evaluation/constants.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+"""
+aqua.evaluation.const
+~~~~~~~~~~~~~~
+
+This module contains constants/enums used in Aqua Evaluation.
+"""
+
+from oci.data_science.models import JobRun
+
+from ads.common.extended_enum import ExtendedEnum
+
+EVAL_TERMINATION_STATE = [
+    JobRun.LIFECYCLE_STATE_SUCCEEDED,
+    JobRun.LIFECYCLE_STATE_FAILED,
+]
+
+
+class EvaluationCustomMetadata(ExtendedEnum):
+    EVALUATION_SOURCE = "evaluation_source"
+    EVALUATION_JOB_ID = "evaluation_job_id"
+    EVALUATION_JOB_RUN_ID = "evaluation_job_run_id"
+    EVALUATION_OUTPUT_PATH = "evaluation_output_path"
+    EVALUATION_SOURCE_NAME = "evaluation_source_name"
+    EVALUATION_ERROR = "aqua_evaluate_error"
+
+
+class EvaluationConfig(ExtendedEnum):
+    PARAMS = "model_params"
+
+
+class EvaluationReportJson(ExtendedEnum):
+    """Contains evaluation report.json fields name."""
+
+    METRIC_SUMMARY_RESULT = "metric_summary_result"
+    METRIC_RESULT = "metric_results"
+    MODEL_PARAMS = "model_params"
+    MODEL_DETAILS = "model_details"
+    DATA = "data"
+    DATASET = "dataset"
+
+
+class EvaluationMetricResult(ExtendedEnum):
+    """Contains metric result's fields name in report.json."""
+
+    SHORT_NAME = "key"
+    NAME = "name"
+    DESCRIPTION = "description"
+    SUMMARY_DATA = "summary_data"
+    DATA = "data"

ads/aqua/evaluation/entities.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+# Copyright (c) 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+"""
+aqua.evaluation.entities
+~~~~~~~~~~~~~~
+
+This module contains dataclasses for aqua evaluation.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import Field
+
+from ads.aqua.config.utils.serializer import Serializable
+from ads.aqua.data import AquaResourceIdentifier
+
+
+class CreateAquaEvaluationDetails(Serializable):
+    """Class for creating aqua model evaluation.
+
+    Properties
+    ----------
+    evaluation_source_id: str
+        The evaluation source id. Must be either model or model deployment ocid.
+    evaluation_name: str
+        The name for evaluation.
+    dataset_path: str
+        The dataset path for the evaluation. Could be either a local path from notebook session
+        or an object storage path.
+    report_path: str
+        The report path for the evaluation. Must be an object storage path.
+    model_parameters: dict
+        The parameters for the evaluation.
+    shape_name: str
+        The shape name for the evaluation job infrastructure.
+    memory_in_gbs: float
+        The memory in gbs for the shape selected.
+    ocpus: float
+        The ocpu count for the shape selected.
+    block_storage_size: int
+        The storage for the evaluation job infrastructure.
+    compartment_id: (str, optional). Defaults to `None`.
+        The compartment id for the evaluation.
+    project_id: (str, optional). Defaults to `None`.
+        The project id for the evaluation.
+    evaluation_description: (str, optional). Defaults to `None`.
+        The description for evaluation
+    experiment_id: (str, optional). Defaults to `None`.
+        The evaluation model version set id. If provided,
+        evaluation model will be associated with it.
+    experiment_name: (str, optional). Defaults to `None`.
+        The evaluation model version set name. If provided,
+        the model version set with the same name will be used if exists,
+        otherwise a new model version set will be created with the name.
+    experiment_description: (str, optional). Defaults to `None`.
+        The description for the evaluation model version set.
+    log_group_id: (str, optional). Defaults to `None`.
+        The log group id for the evaluation job infrastructure.
+    log_id: (str, optional). Defaults to `None`.
+        The log id for the evaluation job infrastructure.
+    metrics: (list, optional). Defaults to `None`.
+        The metrics for the evaluation.
+    force_overwrite: (bool, optional). Defaults to `False`.
+        Whether to force overwrite the existing file in object storage.
+    freeform_tags: (dict, optional)
+        Freeform tags for the evaluation model
+    defined_tags: (dict, optional)
+        Defined tags for the evaluation model
+    """
+
+    evaluation_source_id: str
+    evaluation_name: str
+    dataset_path: str
+    report_path: str
+    model_parameters: dict
+    shape_name: str
+    block_storage_size: int
+    compartment_id: Optional[str] = None
+    project_id: Optional[str] = None
+    evaluation_description: Optional[str] = None
+    experiment_id: Optional[str] = None
+    experiment_name: Optional[str] = None
+    experiment_description: Optional[str] = None
+    memory_in_gbs: Optional[float] = None
+    ocpus: Optional[float] = None
+    log_group_id: Optional[str] = None
+    log_id: Optional[str] = None
+    metrics: Optional[List[Dict[str, Any]]] = None
+    force_overwrite: Optional[bool] = False
+    freeform_tags: Optional[dict] = None
+    defined_tags: Optional[dict] = None
+    container_image_uri: Optional[str] = Field(
+        None,
+        description="Image URI for evaluation container runtime. "
+        "The service managed container will be used by default.",
+    )
+
+    class Config:
+        extra = "ignore"
+        protected_namespaces = ()
+
+
+class AquaEvalReport(Serializable):
+    evaluation_id: str = ""
+    content: str = ""
+
+    class Config:
+        extra = "ignore"
+
+
+class AquaEvalParams(Serializable):
+    shape: str = ""
+    dataset_path: str = ""
+    report_path: str = ""
+
+    class Config:
+        extra = "allow"
+
+
+class AquaEvalMetric(Serializable):
+    key: str
+    name: str
+    description: str = ""
+
+    class Config:
+        extra = "ignore"
+
+
+class AquaEvalMetricSummary(Serializable):
+    metric: str = ""
+    score: str = ""
+    grade: str = ""
+
+    class Config:
+        extra = "ignore"
+
+
+class AquaEvalMetrics(Serializable):
+    id: str
+    report: str
+    metric_results: List[AquaEvalMetric] = Field(default_factory=list)
+    metric_summary_result: List[AquaEvalMetricSummary] = Field(default_factory=list)
+
+    class Config:
+        extra = "ignore"
+
+
+class AquaEvaluationCommands(Serializable):
+    evaluation_id: str
+    evaluation_target_id: str
+    input_data: Dict[str, Any]
+    metrics: List[Dict[str, Any]]
+    output_dir: str
+    params: Dict[str, Any]
+
+    class Config:
+        extra = "ignore"
+
+
+class AquaEvaluationSummary(Serializable):
+    """Represents a summary of Aqua evalution."""
+
+    id: str
+    name: str
+    console_url: str
+    lifecycle_state: str
+    lifecycle_details: str
+    time_created: str
+    tags: Dict[str, Any]
+    experiment: AquaResourceIdentifier = Field(default_factory=AquaResourceIdentifier)
+    source: AquaResourceIdentifier = Field(default_factory=AquaResourceIdentifier)
+    job: AquaResourceIdentifier = Field(default_factory=AquaResourceIdentifier)
+    parameters: AquaEvalParams = Field(default_factory=AquaEvalParams)
+
+    class Config:
+        extra = "ignore"
+
+
+class AquaEvaluationDetail(AquaEvaluationSummary):
+    """Represents a details of Aqua evalution."""
+
+    log_group: AquaResourceIdentifier = Field(default_factory=AquaResourceIdentifier)
+    log: AquaResourceIdentifier = Field(default_factory=AquaResourceIdentifier)
+    introspection: dict = Field(default_factory=dict)

ads/aqua/evaluation/errors.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+"""
+aqua.evaluation.errors
+~~~~~~~~~~~~~~
+
+This module contains errors in Aqua Evaluation.
+"""
+
+from ads.common.extended_enum import ExtendedEnum
+
+
+class EvaluationJobExitCode(ExtendedEnum):
+    SUCCESS = 0
+    COMMON_ERROR = 1
+
+    # Configuration-related issues 10-19
+    INVALID_EVALUATION_CONFIG = 10
+    EVALUATION_CONFIG_NOT_PROVIDED = 11
+    INVALID_OUTPUT_DIR = 12
+    INVALID_INPUT_DATASET_PATH = 13
+    INVALID_EVALUATION_ID = 14
+    INVALID_TARGET_EVALUATION_ID = 15
+    INVALID_EVALUATION_CONFIG_VALIDATION = 16
+
+    # Evaluation process issues 20-39
+    OUTPUT_DIR_NOT_FOUND = 20
+    INVALID_INPUT_DATASET = 21
+    INPUT_DATA_NOT_FOUND = 22
+    EVALUATION_ID_NOT_FOUND = 23
+    EVALUATION_ALREADY_PERFORMED = 24
+    EVALUATION_TARGET_NOT_FOUND = 25
+    NO_SUCCESS_INFERENCE_RESULT = 26
+    COMPUTE_EVALUATION_ERROR = 27
+    EVALUATION_REPORT_ERROR = 28
+    MODEL_INFERENCE_WRONG_RESPONSE_FORMAT = 29
+    UNSUPPORTED_METRICS = 30
+    METRIC_CALCULATION_FAILURE = 31
+    EVALUATION_MODEL_CATALOG_RECORD_CREATION_FAILED = 32
+
+
+EVALUATION_JOB_EXIT_CODE_MESSAGE = {
+    EvaluationJobExitCode.SUCCESS: "Success",
+    EvaluationJobExitCode.COMMON_ERROR: "An error occurred during the evaluation, please check the log for more information.",
+    EvaluationJobExitCode.INVALID_EVALUATION_CONFIG: "The provided evaluation configuration was not in the correct format, supported formats are YAML or JSON.",
+    EvaluationJobExitCode.EVALUATION_CONFIG_NOT_PROVIDED: "The evaluation config was not provided.",
+    EvaluationJobExitCode.INVALID_OUTPUT_DIR: "The specified output directory path is invalid.",
+    EvaluationJobExitCode.INVALID_INPUT_DATASET_PATH: "Dataset path is invalid.",
+    EvaluationJobExitCode.INVALID_EVALUATION_ID: "Evaluation ID was not found in the Model Catalog.",
+    EvaluationJobExitCode.INVALID_TARGET_EVALUATION_ID: "Target evaluation ID was not found in the Model Deployment.",
+    EvaluationJobExitCode.INVALID_EVALUATION_CONFIG_VALIDATION: "Validation errors in the evaluation config.",
+    EvaluationJobExitCode.OUTPUT_DIR_NOT_FOUND: "Destination folder does not exist or cannot be used for writing, verify the folder's existence and permissions.",
+    EvaluationJobExitCode.INVALID_INPUT_DATASET: "Input dataset is in an invalid format, ensure the dataset is in jsonl format and that includes the required columns: 'prompt', 'completion' (optional 'category').",
+    EvaluationJobExitCode.INPUT_DATA_NOT_FOUND: "Input data file does not exist or cannot be use for reading, verify the file's existence and permissions.",
+    EvaluationJobExitCode.EVALUATION_ID_NOT_FOUND: "Evaluation ID does not match any resource in the Model Catalog, or access may be blocked by policies.",
+    EvaluationJobExitCode.EVALUATION_ALREADY_PERFORMED: "Evaluation already has an attached artifact, indicating that the evaluation has already been performed.",
+    EvaluationJobExitCode.EVALUATION_TARGET_NOT_FOUND: "Target evaluation ID does not match any resources in Model Deployment.",
+    EvaluationJobExitCode.NO_SUCCESS_INFERENCE_RESULT: "Inference process completed without producing expected outcome, verify the model parameters and config.",
+    EvaluationJobExitCode.COMPUTE_EVALUATION_ERROR: "Evaluation process encountered an issue while calculating metrics.",
+    EvaluationJobExitCode.EVALUATION_REPORT_ERROR: "Failed to save the evaluation report due to an error. Ensure the evaluation model is currently active and the specified path for the output report is valid and accessible. Verify these conditions and reinitiate the evaluation process.",
+    EvaluationJobExitCode.MODEL_INFERENCE_WRONG_RESPONSE_FORMAT: "Evaluation encountered unsupported, or unexpected model output, verify the target evaluation model is compatible and produces the correct format.",
+    EvaluationJobExitCode.UNSUPPORTED_METRICS: "None of the provided metrics are supported by the framework.",
+    EvaluationJobExitCode.METRIC_CALCULATION_FAILURE: "All attempted metric calculations were unsuccessful. Please review the metric configurations and input data.",
+    EvaluationJobExitCode.EVALUATION_MODEL_CATALOG_RECORD_CREATION_FAILED: (
+        "Failed to create a Model Catalog record for the evaluation. "
+        "This could be due to missing required permissions. "
+        "Please check the log for more information."
+    ),
+}