google-cloud-pipeline-components 2.19.0__py3-none-any.whl → 2.20.0__py3-none-any.whl

google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/autosxs_pipeline.py

@@ -1,109 +0,0 @@
-# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Optimization AI Inference and AutoSxS pipeline function."""
-
-from typing import Any, Dict, List
-
-from google_cloud_pipeline_components import _placeholders
-from google_cloud_pipeline_components._implementation.llm import batch_prediction_pairwise
-from google_cloud_pipeline_components._implementation.llm import model_evaluation_text_generation_pairwise
-from google_cloud_pipeline_components._implementation.llm import online_evaluation_pairwise
-from kfp import dsl
-
-
-# pylint: disable=dangerous-default-value,g-bare-generic,unused-argument
-@dsl.pipeline(
-    name='autosxs-template',
-    description='Determines the SxS winrate between two models.',
-)
-def autosxs_pipeline(
-    evaluation_dataset: str,
-    task: str,
-    id_columns: List[str],
-    model_a: str = '',
-    model_b: str = '',
-    autorater_prompt_parameters: Dict[str, Dict[str, str]] = {},
-    model_a_prompt_parameters: Dict[str, Dict[str, str]] = {},
-    model_b_prompt_parameters: Dict[str, Dict[str, str]] = {},
-    response_column_a: str = '',
-    response_column_b: str = '',
-    model_a_parameters: Dict[str, str] = {},
-    model_b_parameters: Dict[str, str] = {},
-    human_preference_column: str = '',
-    project: str = _placeholders.PROJECT_ID_PLACEHOLDER,
-    location: str = _placeholders.LOCATION_PLACEHOLDER,
-    judgments_format: str = 'jsonl',
-    bigquery_destination_prefix: str = '',
-    experimental_args: Dict[str, Any] = {},
-):
-  # fmt: off
-  """Evaluates two models side-by-side using an arbiter model.
-
-  Args:
-    evaluation_dataset: A BigQuery table or comma-separated list of GCS paths to a JSONL dataset containing evaluation examples.
-    task: Evaluation task in the form `{task}@{version}`. task can be one of `[summarization, question_answering]`. Version is an integer with 3 digits or "latest". Ex: `summarization@001` or `question_answering@latest`.
-    id_columns: The columns which distinguish unique evaluation examples.
-    model_a: A fully-qualified model resource name (`projects/{project}/locations/{location}/models/{model}@{version}`) or publisher model resource name (`publishers/{publisher}/models/{model}`).  This parameter is optional if Model A responses are specified.
-    model_b: A fully-qualified model resource name (`projects/{project}/locations/{location}/models/{model}@{version}`) or publisher model resource name (`publishers/{publisher}/models/{model}`).  This parameter is optional if Model B responses are specified.
-    autorater_prompt_parameters: Map of autorater prompt parameters to columns or templates. The expected parameters are: `inference_instruction` (details on how to perform a task) and `inference_context` (content to reference to perform the task). As an example, `{'inference_context': {'column': 'my_prompt'}}` uses the evaluation dataset's `my_prompt` column for the AutoRater's context.
-    model_a_prompt_parameters: Map of Model A prompt template parameters to columns or templates. This parameter is optional if Model A predictions are predefined. Example - `{'prompt': {'column': 'my_prompt'}}` uses the evaluation dataset's `my_prompt` column for the prompt parameter named `prompt`.
-    model_b_prompt_parameters: Map of Model B prompt template parameters to columns or templates. This parameter is optional if Model B predictions are predefined. Example - `{'prompt': {'column': 'my_prompt'}}` uses the evaluation dataset's `my_prompt` column for the prompt parameter named `prompt`.
-    response_column_a: Either the name of a column in the evaluation dataset containing predefined predictions, or the name of the column in the Model A output containing predictions. If no value is provided, the correct model output column name will attempt to be inferred.
-    response_column_b: Either the name of a column in the evaluation dataset containing predefined predictions, or the name of the column in the Model B output containing predictions. If no value is provided, the correct model output column name will attempt to be inferred.
-    model_a_parameters: The parameters that govern the predictions from model A, such as temperature or maximum output tokens.
-    model_b_parameters: The parameters that govern the predictions from model B, such as temperature or maximum output tokens.
-    human_preference_column: The column containing ground truth winners for each example. Providing this parameter adds additional metrics for checking the AutoRater alignment with human preferences.
-    project: Project used to run custom jobs. This should be the same project used to run the pipeline.
-    location: Location used to run custom jobs. This should be the same location used to run the pipeline.
-    judgments_format: The format to write judgments to. Can be either `[json, bigquery]`.
-    bigquery_destination_prefix: BigQuery table to write judgments to if the specified format is 'bigquery'.
-    experimental_args: Experimentally released arguments. Subject to change.
-  """
-  # fmt: on
-  responses = batch_prediction_pairwise.batch_prediction_pairwise(
-      display_name='autosxs-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}',
-      evaluation_dataset=evaluation_dataset,
-      id_columns=id_columns,
-      task=task,
-      autorater_prompt_parameters=autorater_prompt_parameters,
-      response_column_a=response_column_a,
-      response_column_b=response_column_b,
-      model_a=model_a,
-      model_b=model_b,
-      model_a_prompt_parameters=model_a_prompt_parameters,
-      model_b_prompt_parameters=model_b_prompt_parameters,
-      model_a_parameters=model_a_parameters,
-      model_b_parameters=model_b_parameters,
-      human_preference_column=human_preference_column,
-      experimental_args=experimental_args,
-  ).set_display_name('AutoSxS Batch Prediction')
-
-  winners = online_evaluation_pairwise.online_evaluation_pairwise(
-      inference_output_uri=responses.outputs[
-          'preprocessed_evaluation_dataset_uri'
-      ],
-      id_columns=id_columns,
-      human_preference_column=human_preference_column,
-      task=task,
-      judgments_format=judgments_format,
-      bigquery_destination_prefix=bigquery_destination_prefix,
-      experimental_args=experimental_args,
-  ).set_display_name('AutoSxS Autorater')
-
-  model_evaluation_text_generation_pairwise.model_evaluation_text_generation_pairwise(
-      judgments_dir=winners.outputs['judgments_uri'],
-      human_preference_column=human_preference_column,
-  ).set_display_name(
-      'AutoSxS Metrics'
-  )

google_cloud_pipeline_components/proto/preflight_validations_pb2.py

@@ -1,58 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# Protobuf Python Version: 0.20240110.0
-"""Generated protocol buffer code."""
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf.internal import builder as _builder
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n\x13preflight_validations.proto\x12\x15preflight_validations"\x90\x02\n\x0eValidationItem\x12O\n\x0bsa_metadata\x18\x02'
-    b' \x01(\x0b\x32\x38.preflight_validations.GoogleCloudServiceAccountMetadataH\x00\x12P\n\x0equota_metadata\x18\x03'
-    b' \x01(\x0b\x32\x36.preflight_validations.GoogleCloudProjectQuotaMetadataH\x00\x12O\n\x0c\x61pi_metadata\x18\x04'
-    b' \x01(\x0b\x32\x37.preflight_validations.GoogleCloudApiEnablementMetadataH\x00\x42\n\n\x08metadata"\xeb\x01\n\x1fGoogleCloudProjectQuotaMetadata\x12\x14\n\x0cservice_name\x18\x01'
-    b' \x01(\t\x12s\n\x17metrics_recommendations\x18\x02'
-    b' \x03(\x0b\x32R.preflight_validations.GoogleCloudProjectQuotaMetadata.MetricsRecommendationsEntry\x1a=\n\x1bMetricsRecommendationsEntry\x12\x0b\n\x03key\x18\x01'
-    b' \x01(\t\x12\r\n\x05value\x18\x02'
-    b' \x01(\x03:\x02\x38\x01"P\n!GoogleCloudServiceAccountMetadata\x12\x16\n\x0eprincipal_name\x18\x01'
-    b' \x01(\t\x12\x13\n\x0bpermissions\x18\x02 \x03(\t"9\n'
-    b' GoogleCloudApiEnablementMetadata\x12\x15\n\rservice_names\x18\x01'
-    b' \x03(\tB\x02P\x01\x62\x06proto3'
-)
-
-_globals = globals()
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(
-    DESCRIPTOR,
-    'google_cloud_pipeline_components.google_cloud_pipeline_components.proto.preflight_validations_pb2',
-    _globals,
-)
-if not _descriptor._USE_C_DESCRIPTORS:
-  _globals['DESCRIPTOR']._loaded_options = None
-  _globals['DESCRIPTOR']._serialized_options = b'P\001'
-  _globals[
-      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
-  ]._loaded_options = None
-  _globals[
-      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
-  ]._serialized_options = b'8\001'
-  _globals['_VALIDATIONITEM']._serialized_start = 142
-  _globals['_VALIDATIONITEM']._serialized_end = 414
-  _globals['_GOOGLECLOUDPROJECTQUOTAMETADATA']._serialized_start = 417
-  _globals['_GOOGLECLOUDPROJECTQUOTAMETADATA']._serialized_end = 652
-  _globals[
-      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
-  ]._serialized_start = 591
-  _globals[
-      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
-  ]._serialized_end = 652
-  _globals['_GOOGLECLOUDSERVICEACCOUNTMETADATA']._serialized_start = 654
-  _globals['_GOOGLECLOUDSERVICEACCOUNTMETADATA']._serialized_end = 734
-  _globals['_GOOGLECLOUDAPIENABLEMENTMETADATA']._serialized_start = 736
-  _globals['_GOOGLECLOUDAPIENABLEMENTMETADATA']._serialized_end = 793
-# @@protoc_insertion_point(module_scope)