google-cloud-pipeline-components 2.10.0__py3-none-any.whl → 2.12.0__py3-none-any.whl

google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py

@@ -41,7 +41,7 @@ def model_evaluation_text_generation(
     ground_truth_gcs_source: str = '',
     enable_row_based_metrics: bool = False,
     display_name: str = 'model-evaluation-text-generation',
-    machine_type: str = 'e2-highmem-16',
+    machine_type: str = 'e2-standard-4',
     service_account: str = '',
     network: str = '',
     reserved_ip_ranges: List[str] = [],
@@ -78,7 +78,7 @@ def model_evaluation_text_generation(
       only ground truth files to be used for this evaluation.
     display_name: The name of the evaluation custom job.
     machine_type: The machine type of this custom job. If not set, defaulted to
-      `e2-highmem-16`. More details:
+      `e2-standard-4`. More details:
       https://cloud.google.com/compute/docs/machine-resource
     service_account: Sets the default service account for workload run-as
       account. The service account running the pipeline

google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/component.py

@@ -110,7 +110,7 @@ def llm_evaluation_dataset_preprocessor_graph_component(
     gcs_source_uris: List[str],
     input_field_name: str = 'input_text',
     display_name: str = 'llm_evaluation_dataset_preprocessor_component',
-    machine_type: str = 'e2-highmem-16',
+    machine_type: str = 'e2-standard-4',
     service_account: str = '',
     network: str = '',
     encryption_spec_key_name: str = '',
@@ -128,7 +128,7 @@ def llm_evaluation_dataset_preprocessor_graph_component(
         contains the input prompts to the LLM.
       display_name: The name of the Evaluation job.
       machine_type: The machine type of this custom job. If not set, defaulted
-        to `e2-highmem-16`. More details:
+        to `e2-standard-4`. More details:
         https://cloud.google.com/compute/docs/machine-resource
       service_account: Sets the default service account for workload run-as
         account. The service account running the pipeline

google_cloud_pipeline_components/_implementation/model_evaluation/model_name_preprocessor/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model name preprocessor Component."""

google_cloud_pipeline_components/_implementation/model_evaluation/model_name_preprocessor/component.py

@@ -0,0 +1,74 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model name preprocessor component used in KFP pipelines."""
+
+from google_cloud_pipeline_components._implementation.model_evaluation import version
+from kfp.dsl import container_component
+from kfp.dsl import ContainerSpec
+from kfp.dsl import OutputPath
+from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER
+
+
+@container_component
+def model_name_preprocessor(
+    gcp_resources: OutputPath(str),
+    processed_model_name: OutputPath(str),
+    project: str,
+    location: str,
+    model_name: str,
+    service_account: str = '',
+):
+  """Preprocess inputs for text2sql evaluation pipeline.
+
+  Args:
+      project: Required. The GCP project that runs the pipeline component.
+      location: Required. The GCP region that runs the pipeline component.
+      model_name: The Model name used to run evaluation. Must be a publisher
+        Model or a managed Model sharing the same ancestor location. Starting
+        this job has no impact on any existing deployments of the Model and
+        their resources.
+      service_account: Sets the default service account for workload run-as
+        account. The service account running the pipeline
+        (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account)
+
+  Returns:
+      gcp_resources (str):
+        Serialized gcp_resources proto tracking the custom job.
+      processed_model_name (str):
+        Preprocessed model name.
+  """
+
+  return ContainerSpec(
+      image=version.LLM_EVAL_IMAGE_TAG,
+      args=[
+          '--model_name_preprocessor',
+          'true',
+          '--project',
+          project,
+          '--location',
+          location,
+          '--root_dir',
+          f'{PIPELINE_ROOT_PLACEHOLDER}',
+          '--model_name',
+          model_name,
+          '--processed_model_name',
+          processed_model_name,
+          '--service_account',
+          service_account,
+          '--gcp_resources',
+          gcp_resources,
+          '--executor_input',
+          '{{$}}',
+      ],
+  )

google_cloud_pipeline_components/_implementation/model_evaluation/version.py

@@ -14,7 +14,7 @@
 """Version constants for model evaluation components."""
 
 _EVAL_VERSION = 'v0.9.4'
-_LLM_EVAL_VERSION = 'v0.5'
+_LLM_EVAL_VERSION = 'v0.6'
 
 _EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/model-evaluation'
 _LLM_EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/llm-model-evaluation'

google_cloud_pipeline_components/container/_implementation/model_evaluation/import_model_evaluation.py

@@ -338,13 +338,13 @@ def main(argv):
           and slice_spec['dimension'] == 'annotationSpec'
       ):
         slice_config['model_explanation'] = {
-            'mean_attributions': [
-                {
-                    'feature_attributions': sliced_feature_attributions[
-                        slice_spec['value']
-                    ]
-                }
-            ]
+            'mean_attributions': [{
+                'feature_attributions': (
+                    sliced_feature_attributions[slice_spec['value']]
+                    if slice_spec['value'] in sliced_feature_attributions
+                    else None
+                )
+            }]
         }
         slices_with_explanations.append(slice_config)
       elif 'slice_spec' in slice_spec:

google_cloud_pipeline_components/preview/llm/infer/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

google_cloud_pipeline_components/preview/llm/infer/component.py

@@ -41,7 +41,9 @@ def infer_pipeline(
     sampling_strategy: str = 'greedy',
     instruction: Optional[str] = None,
     project: str = _placeholders.PROJECT_ID_PLACEHOLDER,
+    accelerator_type: str = 'GPU',
     location: str = _placeholders.LOCATION_PLACEHOLDER,
+    encryption_spec_key_name: str = '',
 ) -> PipelineOutput:
   # fmt: off
   """Uses a large-language model to perform bulk inference on a prompt dataset.
@@ -55,7 +57,9 @@ def infer_pipeline(
     sampling_strategy: This field specifies the sampling strategy. The valid options are 'greedy' and 'temperature_sampling'.
     instruction: This field lets the model know what task it needs to perform. Base models have been trained over a large set of varied instructions. You can give a simple and intuitive description of the task and the model will follow it, e.g. "Classify this movie review as positive or negative" or "Translate this sentence to Danish". Do not specify this if your dataset already prepends the instruction to the inputs field.
     project: Project used to run custom jobs. If not specified the project used to run the pipeline will be used.
-    location: Location used to run custom jobs. If not specified the location used to run the pipeline will be used.
+    accelerator_type: One of 'TPU' or 'GPU'. If 'TPU' is specified, tuning components run in europe-west4. Otherwise tuning components run in us-central1 on GPUs. Default is 'GPU'.
+    location: Location used to run non-tuning components, i.e. components that do not require accelerators. If not specified the location used to run the pipeline will be used.
+    encryption_spec_key_name: Customer-managed encryption key. If this is set, then all resources created by the CustomJob will be encrypted with the provided encryption key. Note that this is not supported for TPU at the moment.
 
   Returns:
     Cloud storage path to output predictions.
@@ -63,7 +67,7 @@ def infer_pipeline(
   # fmt: on
   prompt_column = 'input_text'
   machine_spec = function_based.resolve_machine_spec(
-      location=location,
+      accelerator_type=accelerator_type,
       use_test_spec=env.get_use_test_machine_spec(),
   ).set_display_name('Resolve Machine Spec')
   reference_model_metadata = function_based.resolve_reference_model_metadata(
@@ -82,9 +86,6 @@ def infer_pipeline(
       large_model_reference=large_model_reference,
       instruction=instruction,
   ).set_display_name('Resolve Instruction')
-  prompt_dataset_image_uri = function_based.resolve_private_image_uri(
-      image_name='text_importer',
-  ).set_display_name('Resolve Prompt Dataset Image URI')
   prompt_dataset_importer = (
       private_text_importer.private_text_importer(
           project=project,
@@ -96,21 +97,19 @@ def infer_pipeline(
           large_model_reference=reference_model_metadata.outputs[
               'large_model_reference'
           ],
-          image_uri=prompt_dataset_image_uri.output,
           instruction=resolved_text_instruction.output,
+          encryption_spec_key_name=encryption_spec_key_name,
       )
       .set_display_name('Import Prompt Dataset')
       .set_caching_options(False)
   )
 
-  bulk_inferrer_image_uri = function_based.resolve_private_image_uri(
-      image_name='infer',
+  bulk_inferrer_image_uri = function_based.resolve_private_refined_image_uri(
       accelerator_type=machine_spec.outputs['accelerator_type'],
-      accelerator_count=machine_spec.outputs['accelerator_count'],
   ).set_display_name('Resolve Bulk Inferrer Image URI')
   bulk_inference = bulk_inferrer.bulk_inferrer(
       project=project,
-      location=location,
+      location=machine_spec.outputs['tuning_location'],
       input_model=reference_model_metadata.outputs['reference_model_path'],
       input_dataset_path=prompt_dataset_importer.outputs['imported_data_path'],
       dataset_split=env.TRAIN_SPLIT,
@@ -124,6 +123,7 @@ def infer_pipeline(
       accelerator_count=machine_spec.outputs['accelerator_count'],
       machine_type=machine_spec.outputs['machine_type'],
       image_uri=bulk_inferrer_image_uri.output,
+      encryption_spec_key_name=encryption_spec_key_name,
   ).set_display_name('Bulk Inferrer')
 
   return PipelineOutput(

google_cloud_pipeline_components/preview/llm/rlaif/component.py

@@ -40,6 +40,7 @@ def rlaif_pipeline(
     prompt_dataset: str,
     preference_prompt_dataset: str,
     large_model_reference: str,
+    task_type: str,
     model_display_name: Optional[str] = None,
     prompt_sequence_length: int = 512,
     target_sequence_length: int = 64,
@@ -54,6 +55,7 @@ def rlaif_pipeline(
     instruction: Optional[str] = None,
     eval_dataset: Optional[str] = None,
     project: str = _placeholders.PROJECT_ID_PLACEHOLDER,
+    accelerator_type: str = 'GPU',
     location: str = _placeholders.LOCATION_PLACEHOLDER,
     tensorboard_resource_id: Optional[str] = None,
 ) -> PipelineOutput:
@@ -64,7 +66,9 @@ def rlaif_pipeline(
 
   Args:
     prompt_dataset: Cloud storage path to an unlabled JSONL dataset that contains prompts. Text datasets must contain an `input_text` field that contains the prompt. Chat datasets must contain at least 1 message in a `messages` field. Each message must be valid JSON that contains `author` and `content` fields, where valid `author` values are `user` and `assistant` and `content` must be non-empty. Each row may contain multiple messages, but the first and last author must be the `user`. An optional `context` field may be provided for each example in a chat dataset. If provided, the `context` will preprended to the message `content`. The `instruction` serves as the default context. (Useful if most messages use the same system-level context.) Any context provided in the example will override the default value.
-    preference_prompt_dataset: The prompt dataset used for two models' inferences to build the side by side comparison AI feedback. large_model_reference: Name of the base model. Supported values are `text-bison@001`, `t5-small`, `t5-large`, `t5-xl` and `t5-xxl`. `text-bison@001` and `t5-small` are supported in `us-central1` and `europe-west4`. `t5-large`, `t5-xl` and `t5-xxl` are only supported in `europe-west4`.
+    preference_prompt_dataset: The prompt dataset used for two models' inferences to build the side by side comparison AI feedback.
+    large_model_reference: Name of the base model. Supported values are `text-bison@001`, `t5-small`, `t5-large`, `t5-xl` and `t5-xxl`. `text-bison@001` and `t5-small` are supported in `us-central1` and `europe-west4`. `t5-large`, `t5-xl` and `t5-xxl` are only supported in `europe-west4`.
+    task_type: Evaluation task in the form {task}@{version}. task can be one of "summarization", "question_answering". Version is an integer with 3 digits or "latest". Ex: summarization@001 or question_answering@latest.
     model_display_name: Name of the fine-tuned model shown in the Model Registry. If not provided, a default name will be created.
     prompt_sequence_length: Maximum tokenized sequence length for input text. Higher values increase memory overhead. This value should be at most 8192. Default value is 512.
     target_sequence_length:  Maximum tokenized sequence length for target text. Higher values increase memory overhead. This value should be at most 1024. Default value is 64.
@@ -79,6 +83,7 @@ def rlaif_pipeline(
     instruction: This field lets the model know what task it needs to perform. Base models have been trained over a large set of varied instructions. You can give a simple and intuitive description of the task and the model will follow it, e.g., "Classify this movie review as positive or negative" or "Translate this sentence to Danish". Do not specify this if your dataset already prepends the instruction to the inputs field.
     eval_dataset: Optional Cloud storage path to an evaluation dataset. If provided, inference will be performed on this dataset after training. The dataset format is jsonl. Each example in the dataset must contain a field `input_text` that contains the prompt.
     project: Project used to run custom jobs. If not specified the project used to run the pipeline will be used.
+    accelerator_type: One of 'TPU' or 'GPU'. If 'TPU' is specified, tuning components run in europe-west4. Otherwise tuning components run in us-central1 on GPUs. Default is 'GPU'.
     location: Location used to run custom jobs. If not specified the location used to run the pipeline will be used.
     tensorboard_resource_id: Optional tensorboard resource id in format `projects/{project_number}/locations/{location}/tensorboards/{tensorboard_id}`. If provided, tensorboard metrics will be uploaded to this location.
 
@@ -88,7 +93,6 @@ def rlaif_pipeline(
   """
   # fmt: on
   id_columns = ['content']
-  task = 'summarization@001'
   deploy_model = True
 
   output_prediction_gcs_path_a = infer.infer_pipeline(
@@ -100,6 +104,7 @@ def rlaif_pipeline(
       instruction=instruction,
       project=project,
       location=location,
+      accelerator_type=accelerator_type,
   ).set_display_name('Inferrer A')
   output_prediction_gcs_path_b = infer.infer_pipeline(
       large_model_reference=large_model_b_reference,
@@ -110,6 +115,7 @@ def rlaif_pipeline(
       instruction=instruction,
       project=project,
       location=location,
+      accelerator_type=accelerator_type,
   ).set_display_name('Inferrer B')
 
   inference_output_uri = (
@@ -125,7 +131,7 @@ def rlaif_pipeline(
   autosxs = online_evaluation_pairwise.online_evaluation_pairwise(
       inference_output_uri=inference_output_uri,
       id_columns=id_columns,
-      task=task,
+      task=task_type,
   ).set_display_name('Build AI Feedback')
 
   preference_dataset = (
@@ -155,6 +161,7 @@ def rlaif_pipeline(
           project=project,
           location=location,
           tensorboard_resource_id=tensorboard_resource_id,
+          accelerator_type=accelerator_type,
       )
       .set_display_name('Reinforcement Learning From AI Feedback')
       .outputs

google_cloud_pipeline_components/preview/llm/rlhf/component.py

@@ -17,9 +17,11 @@ from typing import NamedTuple, Optional
 
 from google_cloud_pipeline_components import _placeholders
 from google_cloud_pipeline_components._implementation.llm import deployment_graph
+from google_cloud_pipeline_components._implementation.llm import env
 from google_cloud_pipeline_components._implementation.llm import function_based
 from google_cloud_pipeline_components._implementation.llm import reinforcement_learning_graph
 from google_cloud_pipeline_components._implementation.llm import reward_model_graph
+from google_cloud_pipeline_components._implementation.llm import validate_pipeline
 from google_cloud_pipeline_components.preview.llm.infer import component
 import kfp
 
@@ -48,8 +50,10 @@ def rlhf_pipeline(
     deploy_model: bool = True,
     eval_dataset: Optional[str] = None,
     project: str = _placeholders.PROJECT_ID_PLACEHOLDER,
+    accelerator_type: str = 'GPU',
     location: str = _placeholders.LOCATION_PLACEHOLDER,
-    tensorboard_resource_id: Optional[str] = None,
+    encryption_spec_key_name: str = '',
+    tensorboard_resource_id: str = '',
 ) -> PipelineOutput:
   # fmt: off
   """Performs reinforcement learning from human feedback.
@@ -68,9 +72,11 @@ def rlhf_pipeline(
     kl_coeff: Coefficient for KL penalty. This regularizes the policy model and penalizes if it diverges from its initial distribution. If set to 0, the reference language model is not loaded into memory. Default value is 0.1.
     instruction: This field lets the model know what task it needs to perform. Base models have been trained over a large set of varied instructions. You can give a simple and intuitive description of the task and the model will follow it, e.g. "Classify this movie review as positive or negative" or "Translate this sentence to Danish". Do not specify this if your dataset already prepends the instruction to the inputs field.
     deploy_model: Whether to deploy the model to an endpoint in `us-central1`. Default is True.
-    eval_dataset: Optional Cloud storage path to an evaluation dataset. Note, eval dataset can only be provided for third-party models. If provided, inference will be performed on this dataset after training. The dataset format is jsonl. Each example in the dataset must contain a field `input_text` that contains the prompt.
+    eval_dataset: Optional Cloud storage path to an evaluation dataset. The dataset format is jsonl. The evaluation dataset can be used to compute train-time metrics (when training a reward model) or perform bulk inference for third-party models. To compute train-time metrics this dataset must contain the same fields as the peference dataset. For bulk inference with third-party models only `input_text` is needed. Note, train-time metrics are only computed for the first 5000 samples in the dataset for efficient evaluation during training.
     project: Project used to run custom jobs. If not specified the project used to run the pipeline will be used.
-    location: Location used to run custom jobs. If not specified the location used to run the pipeline will be used.
+    accelerator_type: One of 'TPU' or 'GPU'. If 'TPU' is specified, tuning components run in europe-west4. Otherwise tuning components run in us-central1 on GPUs. Default is 'GPU'.
+    location: Location used to run non-tuning components, i.e. components that do not require accelerators. If not specified the location used to run the pipeline will be used.
+    encryption_spec_key_name: Customer-managed encryption key. If this is set, then all resources created by the CustomJob will be encrypted with the provided encryption key. Note that this is not supported for TPU at the moment.
     tensorboard_resource_id: Optional tensorboard resource id in format `projects/{project_number}/locations/{location}/tensorboards/{tensorboard_id}`. If provided, tensorboard metrics will be uploaded to this location.
 
   Returns:
@@ -78,31 +84,40 @@ def rlhf_pipeline(
     endpoint_resource_name: Path the Online Prediction Endpoint. This will be an empty string if the model was not deployed.
   """
   # fmt: on
-
   # LoRA dim for reward model
   reward_lora_dim = 4
 
-  function_based.validate_rlhf_inputs(
-      large_model_reference=large_model_reference,
+  validate_pipeline_task = validate_pipeline.validate_pipeline(
+      accelerator_type=accelerator_type,
+      location=location,
+      encryption_spec_key_name=encryption_spec_key_name,
       eval_dataset=eval_dataset,
   ).set_display_name('Validate Inputs')
 
   reward_model_pipeline = (
-      reward_model_graph.pipeline(
-          preference_dataset=preference_dataset,
-          large_model_reference=large_model_reference,
-          prompt_sequence_length=prompt_sequence_length,
-          target_sequence_length=target_sequence_length,
-          instruction=instruction,
-          reward_model_learning_rate_multiplier=reward_model_learning_rate_multiplier,
-          reward_model_train_steps=reward_model_train_steps,
-          lora_dim=reward_lora_dim,
-          project=project,
-          location=location,
-          tensorboard_resource_id=tensorboard_resource_id,
+      (
+          reward_model_graph.pipeline(
+              preference_dataset=preference_dataset,
+              large_model_reference=large_model_reference,
+              prompt_sequence_length=prompt_sequence_length,
+              target_sequence_length=target_sequence_length,
+              eval_dataset=validate_pipeline_task.outputs[
+                  'reward_model_eval_dataset'
+              ],
+              instruction=instruction,
+              reward_model_learning_rate_multiplier=reward_model_learning_rate_multiplier,
+              reward_model_train_steps=reward_model_train_steps,
+              lora_dim=reward_lora_dim,
+              project=project,
+              location=location,
+              accelerator_type=accelerator_type,
+              tensorboard_resource_id=tensorboard_resource_id,
+              encryption_spec_key_name=encryption_spec_key_name,
+          )
       )
-  ).set_display_name('Train Reward Model')
-
+      .set_display_name('Train Reward Model')
+      .after(validate_pipeline_task)
+  )
   rl_model_pipeline = reinforcement_learning_graph.pipeline(
       prompt_dataset=prompt_dataset,
       input_reward_model_path=reward_model_pipeline.outputs[
@@ -123,8 +138,10 @@ def rlhf_pipeline(
       instruction=instruction,
       reward_lora_dim=reward_lora_dim,
       project=project,
+      accelerator_type=accelerator_type,
       location=location,
       tensorboard_resource_id=tensorboard_resource_id,
+      encryption_spec_key_name=encryption_spec_key_name,
   ).set_display_name('Reinforcement Learning')
 
   has_inference_dataset = function_based.value_exists(
@@ -135,7 +152,7 @@ def rlhf_pipeline(
       name='Perform Inference',
   ):
     has_model_checkpoint = function_based.value_exists(
-        value=rl_model_pipeline.outputs['output_adapter_path']
+        value=rl_model_pipeline.outputs['output_model_path']
     ).set_display_name('Resolve Model Checkpoint')
     with kfp.dsl.Condition(
         has_model_checkpoint.output == True,  # pylint: disable=singleton-comparison
@@ -145,11 +162,13 @@ def rlhf_pipeline(
           project=project,
           location=location,
           large_model_reference=large_model_reference,
-          model_checkpoint=rl_model_pipeline.outputs['output_adapter_path'],
+          model_checkpoint=rl_model_pipeline.outputs['output_model_path'],
           prompt_dataset=eval_dataset,
           prompt_sequence_length=prompt_sequence_length,
           target_sequence_length=target_sequence_length,
           instruction=instruction,
+          accelerator_type=accelerator_type,
+          encryption_spec_key_name=encryption_spec_key_name,
       )
 
   llm_model_handler = deployment_graph.pipeline(
@@ -157,6 +176,8 @@ def rlhf_pipeline(
       large_model_reference=large_model_reference,
       model_display_name=model_display_name,
       deploy_model=deploy_model,
+      encryption_spec_key_name=encryption_spec_key_name,
+      upload_location=location,
   ).set_display_name('Upload and Deploy Tuned Model')
 
   return PipelineOutput(

google_cloud_pipeline_components/preview/model_evaluation/__init__.py

@@ -14,12 +14,12 @@
 """Model evaluation preview components."""
 
 from google_cloud_pipeline_components.preview.model_evaluation.data_bias_component import detect_data_bias as DetectDataBiasOp
-from google_cloud_pipeline_components.preview.model_evaluation.evaluation_llm_classification_pipeline import evaluation_llm_classification_pipeline
-from google_cloud_pipeline_components.preview.model_evaluation.evaluation_llm_text_generation_pipeline import evaluation_llm_text_generation_pipeline
 from google_cloud_pipeline_components.preview.model_evaluation.feature_attribution_component import feature_attribution as ModelEvaluationFeatureAttributionOp
 from google_cloud_pipeline_components.preview.model_evaluation.feature_attribution_graph_component import feature_attribution_graph_component as FeatureAttributionGraphComponentOp
 from google_cloud_pipeline_components.preview.model_evaluation.model_based_llm_evaluation.autosxs.autosxs_pipeline import autosxs_pipeline
 from google_cloud_pipeline_components.preview.model_evaluation.model_bias_component import detect_model_bias as DetectModelBiasOp
+from google_cloud_pipeline_components.v1.model_evaluation.evaluation_llm_classification_pipeline import evaluation_llm_classification_pipeline
+from google_cloud_pipeline_components.v1.model_evaluation.evaluation_llm_text_generation_pipeline import evaluation_llm_text_generation_pipeline
 
 __all__ = [
     'autosxs_pipeline',

google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/autosxs_pipeline.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 """Optimization AI Inference and AutoSxS pipeline function."""
 
-from typing import Any, Dict, List
+from typing import Any, Dict, List, NamedTuple
 
 from google_cloud_pipeline_components import _placeholders
 from google_cloud_pipeline_components._implementation.llm import batch_prediction_pairwise
@@ -21,6 +21,14 @@ from google_cloud_pipeline_components._implementation.llm import model_evaluatio
 from google_cloud_pipeline_components._implementation.llm import online_evaluation_pairwise
 from kfp import dsl
 
+PipelineOutput = NamedTuple(
+    'Outputs',
+    model_a_evaluation_resource_name=str,
+    model_b_evaluation_resource_name=str,
+    evaluation_count=int,
+    evaluation_dataset_path=str,
+)
+
 
 # pylint: disable=dangerous-default-value,g-bare-generic,unused-argument
 @dsl.pipeline(
@@ -46,7 +54,8 @@ def autosxs_pipeline(
     judgments_format: str = 'jsonl',
     bigquery_destination_prefix: str = '',
     experimental_args: Dict[str, Any] = {},
-):
+    encryption_spec_key_name: str = '',
+) -> PipelineOutput:
   # fmt: off
   """Evaluates two models side-by-side using an arbiter model.
 
@@ -69,6 +78,13 @@ def autosxs_pipeline(
     judgments_format: The format to write judgments to. Can be either `[json, bigquery]`.
     bigquery_destination_prefix: BigQuery table to write judgments to if the specified format is 'bigquery'.
     experimental_args: Experimentally released arguments. Subject to change.
+    encryption_spec_key_name: Customer-managed encryption key options. If this is set, then all resources created by the pipeline will be encrypted with the provided encryption key.
+
+  Returns:
+    model_a_evaluation_resource_name: The path to write the ModelEvaluation for Model A to if Model A is a ModelRegistry Model.
+    model_b_evaluation_resource_name: The path to write the ModelEvaluation for Model B to if Model B is a ModelRegistry Model.
+    evaluation_count: The count of how many evaluations were included for this AutoSxS run.
+    evaluation_dataset_path: The path to the overall evaluation dataset including judgments.
   """
   # fmt: on
   responses = batch_prediction_pairwise.batch_prediction_pairwise(
@@ -87,6 +103,9 @@ def autosxs_pipeline(
       model_b_parameters=model_b_parameters,
       human_preference_column=human_preference_column,
       experimental_args=experimental_args,
+      project=project,
+      location=location,
+      encryption_spec_key_name=encryption_spec_key_name,
   ).set_display_name('AutoSxS Batch Prediction')
 
   winners = online_evaluation_pairwise.online_evaluation_pairwise(
@@ -99,11 +118,34 @@ def autosxs_pipeline(
       judgments_format=judgments_format,
       bigquery_destination_prefix=bigquery_destination_prefix,
       experimental_args=experimental_args,
+      project=project,
+      location=location,
+      encryption_spec_key_name=encryption_spec_key_name,
   ).set_display_name('AutoSxS Autorater')
 
-  model_evaluation_text_generation_pairwise.model_evaluation_text_generation_pairwise(
+  metrics = model_evaluation_text_generation_pairwise.model_evaluation_text_generation_pairwise(
       judgments_dir=winners.outputs['judgments_uri'],
       human_preference_column=human_preference_column,
+      project=project,
+      location=location,
+      encryption_spec_key_name=encryption_spec_key_name,
+      model_a=model_a,
+      model_b=model_b,
+      evaluation_dataset=evaluation_dataset,
+      evaluation_dataset_metadata=winners.outputs['metadata'],
+      task=task,
   ).set_display_name(
       'AutoSxS Metrics'
   )
+
+  return PipelineOutput(
+      model_a_evaluation_resource_name=metrics.outputs[
+          'model_a_evaluation_path'
+      ],
+      model_b_evaluation_resource_name=metrics.outputs[
+          'model_b_evaluation_path'
+      ],
+      evaluation_count=metrics.outputs['evaluation_count_path'],
+      # Needs to be a component output
+      evaluation_dataset_path=metrics.outputs['evaluation_dataset_path'],
+  )

google_cloud_pipeline_components/proto/preflight_validations_pb2.py

@@ -12,16 +12,17 @@ _sym_db = _symbol_database.Default()
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n\x13preflight_validations.proto\x12\x15preflight_validations"\x90\x02\n\x0eValidationItem\x12O\n\x0bsa_metadata\x18\x02'
-    b' \x01(\x0b\x32\x38.preflight_validations.GoogleCloudServiceAccountMetadataH\x00\x12P\n\x0equota_metadata\x18\x03'
-    b' \x01(\x0b\x32\x36.preflight_validations.GoogleCloudProjectQuotaMetadataH\x00\x12O\n\x0c\x61pi_metadata\x18\x04'
-    b' \x01(\x0b\x32\x37.preflight_validations.GoogleCloudApiEnablementMetadataH\x00\x42\n\n\x08metadata"\xeb\x01\n\x1fGoogleCloudProjectQuotaMetadata\x12\x14\n\x0cservice_name\x18\x01'
-    b' \x01(\t\x12s\n\x17metrics_recommendations\x18\x02'
-    b' \x03(\x0b\x32R.preflight_validations.GoogleCloudProjectQuotaMetadata.MetricsRecommendationsEntry\x1a=\n\x1bMetricsRecommendationsEntry\x12\x0b\n\x03key\x18\x01'
-    b' \x01(\t\x12\r\n\x05value\x18\x02'
-    b' \x01(\x03:\x02\x38\x01"P\n!GoogleCloudServiceAccountMetadata\x12\x16\n\x0eprincipal_name\x18\x01'
-    b' \x01(\t\x12\x13\n\x0bpermissions\x18\x02 \x03(\t"9\n'
-    b' GoogleCloudApiEnablementMetadata\x12\x15\n\rservice_names\x18\x01'
+    b'\n\x13preflight_validations.proto\x12\x15preflight_validations"\x8e\x02\n\x0fValidationItems\x12R\n\x0esa_validations\x18\x01'
+    b' \x03(\x0b\x32:.preflight_validations.GoogleCloudServiceAccountValidation\x12S\n\x11quota_validations\x18\x02'
+    b' \x03(\x0b\x32\x38.preflight_validations.GoogleCloudProjectQuotaValidation\x12R\n\x0f\x61pi_validations\x18\x03'
+    b' \x03(\x0b\x32\x39.preflight_validations.GoogleCloudApiEnablementValidation"p\n!GoogleCloudProjectQuotaValidation\x12\x13\n\x0bmetric_name\x18\x01'
+    b' \x01(\t\x12\x15\n\x0bint64_value\x18\x02'
+    b' \x01(\x03H\x00\x12\x16\n\x0c\x64ouble_value\x18\x03'
+    b' \x01(\x01H\x00\x42\x07\n\x05value"\x8d\x01\n#GoogleCloudServiceAccountValidation\x12\x1f\n\x17\x64\x65\x66\x61ult_principal_email\x18\x01'
+    b' \x01(\t\x12\x1c\n\x14override_placeholder\x18\x02'
+    b' \x01(\t\x12\x13\n\x0bpermissions\x18\x03'
+    b' \x03(\t\x12\x12\n\nrole_names\x18\x04'
+    b' \x03(\t";\n"GoogleCloudApiEnablementValidation\x12\x15\n\rservice_names\x18\x01'
     b' \x03(\tB\x02P\x01\x62\x06proto3'
 )
 
@@ -35,24 +36,12 @@ _builder.BuildTopDescriptorsAndMessages(
 if not _descriptor._USE_C_DESCRIPTORS:
   _globals['DESCRIPTOR']._loaded_options = None
   _globals['DESCRIPTOR']._serialized_options = b'P\001'
-  _globals[
-      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
-  ]._loaded_options = None
-  _globals[
-      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
-  ]._serialized_options = b'8\001'
-  _globals['_VALIDATIONITEM']._serialized_start = 142
-  _globals['_VALIDATIONITEM']._serialized_end = 414
-  _globals['_GOOGLECLOUDPROJECTQUOTAMETADATA']._serialized_start = 417
-  _globals['_GOOGLECLOUDPROJECTQUOTAMETADATA']._serialized_end = 652
-  _globals[
-      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
-  ]._serialized_start = 591
-  _globals[
-      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
-  ]._serialized_end = 652
-  _globals['_GOOGLECLOUDSERVICEACCOUNTMETADATA']._serialized_start = 654
-  _globals['_GOOGLECLOUDSERVICEACCOUNTMETADATA']._serialized_end = 734
-  _globals['_GOOGLECLOUDAPIENABLEMENTMETADATA']._serialized_start = 736
-  _globals['_GOOGLECLOUDAPIENABLEMENTMETADATA']._serialized_end = 793
+  _globals['_VALIDATIONITEMS']._serialized_start = 142
+  _globals['_VALIDATIONITEMS']._serialized_end = 412
+  _globals['_GOOGLECLOUDPROJECTQUOTAVALIDATION']._serialized_start = 414
+  _globals['_GOOGLECLOUDPROJECTQUOTAVALIDATION']._serialized_end = 526
+  _globals['_GOOGLECLOUDSERVICEACCOUNTVALIDATION']._serialized_start = 529
+  _globals['_GOOGLECLOUDSERVICEACCOUNTVALIDATION']._serialized_end = 670
+  _globals['_GOOGLECLOUDAPIENABLEMENTVALIDATION']._serialized_start = 672
+  _globals['_GOOGLECLOUDAPIENABLEMENTVALIDATION']._serialized_end = 731
 # @@protoc_insertion_point(module_scope)