google-cloud-pipeline-components 2.16.1__py3-none-any.whl → 2.17.0__py3-none-any.whl

google_cloud_pipeline_components/_implementation/llm/generated/refined_image_versions.py

@@ -17,4 +17,4 @@
 DO NOT EDIT - This file is generated, manual changes will be overridden.
 """
 
-IMAGE_TAG = '20240623_1707'
+IMAGE_TAG = '20240818_1707'

google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Google Cloud Pipeline Evaluation Import Evaluation Component."""

google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/component.py

@@ -0,0 +1,208 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import List, Optional
+
+from google_cloud_pipeline_components import _image
+from google_cloud_pipeline_components.types.artifact_types import ClassificationMetrics
+from google_cloud_pipeline_components.types.artifact_types import ForecastingMetrics
+from google_cloud_pipeline_components.types.artifact_types import RegressionMetrics
+from google_cloud_pipeline_components.types.artifact_types import VertexModel
+from kfp import dsl
+from kfp.dsl import Input
+from kfp.dsl import Metrics
+
+
+@dsl.container_component
+def model_evaluation_import(
+    model: Input[VertexModel],
+    gcp_resources: dsl.OutputPath(str),
+    evaluation_resource_name: dsl.OutputPath(str),
+    metrics: Optional[Input[Metrics]] = None,
+    row_based_metrics: Optional[Input[Metrics]] = None,
+    problem_type: Optional[str] = None,
+    classification_metrics: Optional[Input[ClassificationMetrics]] = None,
+    forecasting_metrics: Optional[Input[ForecastingMetrics]] = None,
+    regression_metrics: Optional[Input[RegressionMetrics]] = None,
+    text_generation_metrics: Optional[Input[Metrics]] = None,
+    question_answering_metrics: Optional[Input[Metrics]] = None,
+    summarization_metrics: Optional[Input[Metrics]] = None,
+    explanation: Optional[Input[Metrics]] = None,
+    feature_attributions: Optional[Input[Metrics]] = None,
+    embedding_metrics: Optional[Input[Metrics]] = None,
+    display_name: str = "",
+    dataset_path: str = "",
+    dataset_paths: List[str] = [],
+    dataset_type: str = "",
+):
+  # fmt: off
+  """Imports a model evaluation artifact to an existing Vertex model with
+  ModelService.ImportModelEvaluation.
+
+  For more details, see
+  https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models.evaluations
+  One of the four metrics inputs must be provided, metrics & problem_type,
+  classification_metrics, regression_metrics, or forecasting_metrics.
+
+  Args:
+    model: Vertex model resource that will be the parent resource of the
+      uploaded evaluation.
+    metrics: Path of metrics generated from an evaluation component.
+    row_based_metrics:
+      Path of row_based_metrics generated from an evaluation component.
+    problem_type: The problem type of the metrics being imported to the
+      VertexModel. `classification`, `regression`, `forecasting`,
+      `text-generation`, `question-answering`, and `summarization` are the
+      currently supported problem types. Must be provided when `metrics` is
+      provided.
+    classification_metrics: google.ClassificationMetrics artifact generated from
+      the ModelEvaluationClassificationOp component.
+    forecasting_metrics: google.ForecastingMetrics artifact generated from
+      the ModelEvaluationForecastingOp component.
+    regression_metrics: google.ClassificationMetrics artifact generated from
+      the ModelEvaluationRegressionOp component.
+    text_generation_metrics: system.Metrics artifact generated from
+      the LLMEvaluationTextGenerationOp component. Subject to change to
+      google.TextGenerationMetrics.
+    question_answering_metrics: system.Metrics artifact generated from
+      the LLMEvaluationTextGenerationOp component. Subject to change to
+      google.QuestionAnsweringMetrics.
+    summarization_metrics: system.Metrics artifact generated from
+      the LLMEvaluationTextGenerationOp component. Subject to change to
+      google.SummarizationMetrics.
+    explanation: Path for model explanation metrics generated from an evaluation
+      component.
+    feature_attributions: The feature attributions metrics artifact generated
+      from the feature attribution component.
+    embedding_metrics: The embedding metrics artifact generated from the
+      embedding retrieval metrics component.
+    display_name: The display name for the uploaded model evaluation resource.
+  """
+  # fmt: on
+  return dsl.ContainerSpec(
+      image=_image.GCPC_IMAGE_TAG,
+      command=[
+          "python3",
+          "-u",
+          "-m",
+          "google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation",
+      ],
+      args=[
+          dsl.IfPresentPlaceholder(
+              input_name="metrics",
+              then=[
+                  "--metrics",
+                  metrics.uri,
+                  "--metrics_explanation",
+                  metrics.metadata["explanation_gcs_path"],
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="row_based_metrics",
+              then=[
+                  "--row_based_metrics",
+                  row_based_metrics.uri,
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="explanation",
+              then=[
+                  "--explanation",
+                  explanation.metadata["explanation_gcs_path"],
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="classification_metrics",
+              then=[
+                  "--classification_metrics",
+                  classification_metrics.uri,
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="forecasting_metrics",
+              then=[
+                  "--forecasting_metrics",
+                  forecasting_metrics.uri,
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="regression_metrics",
+              then=[
+                  "--regression_metrics",
+                  regression_metrics.uri,
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="text_generation_metrics",
+              then=[
+                  "--text_generation_metrics",
+                  text_generation_metrics.uri,
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="question_answering_metrics",
+              then=[
+                  "--question_answering_metrics",
+                  question_answering_metrics.uri,
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="summarization_metrics",
+              then=[
+                  "--summarization_metrics",
+                  summarization_metrics.uri,
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="feature_attributions",
+              then=[
+                  "--feature_attributions",
+                  feature_attributions.uri,
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="embedding_metrics",
+              then=[
+                  "--embedding_metrics",
+                  embedding_metrics.uri,
+              ],
+          ),
+          dsl.IfPresentPlaceholder(
+              input_name="problem_type",
+              then=[
+                  "--problem_type",
+                  problem_type,
+              ],
+          ),
+          "--display_name",
+          display_name,
+          "--dataset_path",
+          dataset_path,
+          "--dataset_paths",
+          dataset_paths,
+          "--dataset_type",
+          dataset_type,
+          "--pipeline_job_id",
+          dsl.PIPELINE_JOB_ID_PLACEHOLDER,
+          "--pipeline_job_resource_name",
+          dsl.PIPELINE_JOB_RESOURCE_NAME_PLACEHOLDER,
+          "--model_name",
+          model.metadata["resourceName"],
+          "--gcp_resources",
+          gcp_resources,
+          "--evaluation_resource_name",
+          evaluation_resource_name,
+      ],
+  )

google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py

@@ -32,6 +32,7 @@ def model_evaluation_text_generation(
     row_based_metrics: Output[Metrics],
     project: str,
     location: str,
+    model_name: str,
     evaluation_task: str = 'text-generation',
     target_field_name: str = 'instance.output_text',
     prediction_field_name: str = 'predictions.content',
@@ -55,6 +56,7 @@ def model_evaluation_text_generation(
   Args:
     project: The GCP project that runs the pipeline component.
     location: The GCP region that runs the pipeline component.
+    model_name: The name of the model to be evaluated.
     evaluation_task: The task that the large language model will be evaluated
       on. The evaluation component computes a set of metrics relevant to that
       specific task. Currently supported tasks are: `summarization`,
@@ -124,6 +126,7 @@ def model_evaluation_text_generation(
           machine_type=machine_type,
           image_uri=version.LLM_EVAL_IMAGE_TAG,
           args=[
+              f'--model_name={model_name}',
               f'--evaluation_task={evaluation_task}',
               f'--target_field_name={target_field_name}',
               f'--prediction_field_name={prediction_field_name}',

google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/component.py

@@ -101,10 +101,8 @@ def evaluation_dataset_preprocessor_internal(
               f'--gcs_source_uris={gcs_source_uris}',
               f'--input_field_name={input_field_name}',
               f'--role_field_name={role_field_name}',
-              (
-                  f'--target_field_name={target_field_name}'
-                  f'--model_name={model_name}'
-              ),
+              f'--target_field_name={target_field_name}',
+              f'--model_name={model_name}',
               f'--output_dirs={output_dirs}',
               '--executor_input={{$.json_escape[1]}}',
           ],

google_cloud_pipeline_components/_implementation/model_evaluation/version.py

@@ -14,7 +14,7 @@
 """Version constants for model evaluation components."""
 
 _EVAL_VERSION = 'v0.9.4'
-_LLM_EVAL_VERSION = 'v0.6'
+_LLM_EVAL_VERSION = 'v0.7'
 
 _EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/model-evaluation'
 _LLM_EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/llm-model-evaluation'

google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_classification_pipeline.py

@@ -0,0 +1,180 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Vertex Gen AI Evaluation for text classification task."""
+
+from typing import Dict, List, NamedTuple
+
+from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationClassificationPredictionsPostprocessorOp
+from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationPreprocessorOp
+from google_cloud_pipeline_components._implementation.model_evaluation import ModelImportEvaluationOp
+from google_cloud_pipeline_components.types.artifact_types import ClassificationMetrics
+from google_cloud_pipeline_components.types.artifact_types import VertexModel
+from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp
+from google_cloud_pipeline_components.v1.model_evaluation.classification_component import model_evaluation_classification as ModelEvaluationClassificationOp
+from kfp import dsl
+# pylint: disable=unused-argument, unexpected-keyword-arg
+
+_PIPELINE_NAME = 'evaluation-llm-classification-pipeline'
+
+
+@dsl.pipeline(name=_PIPELINE_NAME)
+def evaluation_llm_classification_pipeline(  # pylint: disable=dangerous-default-value
+    project: str,
+    location: str,
+    target_field_name: str,
+    batch_predict_gcs_source_uris: List[str],
+    batch_predict_gcs_destination_output_uri: str,
+    model_name: str = 'publishers/google/models/text-bison@002',
+    evaluation_task: str = 'text-classification',
+    evaluation_class_labels: List[str] = [],
+    input_field_name: str = 'input_text',
+    batch_predict_instances_format: str = 'jsonl',
+    batch_predict_predictions_format: str = 'jsonl',
+    batch_predict_model_parameters: Dict[str, str] = {},
+    machine_type: str = 'e2-highmem-16',
+    service_account: str = '',
+    network: str = '',
+    dataflow_machine_type: str = 'n1-standard-4',
+    dataflow_disk_size_gb: int = 50,
+    dataflow_max_num_workers: int = 5,
+    dataflow_service_account: str = '',
+    dataflow_subnetwork: str = '',
+    dataflow_use_public_ips: bool = True,
+    encryption_spec_key_name: str = '',
+    evaluation_display_name: str = 'evaluation-llm-classification-pipeline-{{$.pipeline_job_uuid}}',
+) -> NamedTuple(
+    'outputs',
+    evaluation_metrics=ClassificationMetrics,
+    evaluation_resource_name=str,
+):
+  # fmt: off
+  """The LLM Text Classification Evaluation pipeline.
+
+  Args:
+    project: Required. The GCP project that runs the pipeline components.
+    location: Required. The GCP region that runs the pipeline components.
+    target_field_name: Required. The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction.
+    batch_predict_gcs_source_uris: Required. Google Cloud Storage URI(-s) to your instances data to run batch prediction on. The instances data should also contain the ground truth (target) data, used for evaluation. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.
+    batch_predict_gcs_destination_output_uri: Required. The Google Cloud Storage location of the directory where the output is to be written to.
+    model_name: The Model name used to run evaluation. Must be a publisher Model or a managed Model sharing the same ancestor location. Starting this job has no impact on any existing deployments of the Model and their resources.
+    evaluation_task: The task that the large language model will be evaluated on. The evaluation component computes a set of metrics relevant to that specific task. Currently supported Classification tasks is: `text-classification`.
+    evaluation_class_labels: The JSON array of class names for the target_field, in the same order they appear in the batch predictions input file.
+    input_field_name: The field name of the input eval dataset instances that contains the input prompts to the LLM.
+    batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.
+    batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.
+    batch_predict_model_parameters: A map of parameters that govern the predictions. Some acceptable parameters include: maxOutputTokens, topK, topP, and temperature.
+    machine_type: The machine type of the custom jobs in this pipeline. If not set, defaulted to `e2-highmem-16`. More details: https://cloud.google.com/compute/docs/machine-resource
+    service_account: Sets the default service account for workload run-as account. The service account running the pipeline (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code Service Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project.
+    network: The full name of the Compute Engine network to which the job should be peered. For example, `projects/12345/global/networks/myVPC`. Format is of the form `projects/{project}/global/networks/{network}`. Where `{project}` is a project number, as in `12345`, and `{network}` is a network name, as in `myVPC`. To specify this field, you must have already configured VPC Network Peering for Vertex AI (https://cloud.google.com/vertex-ai/docs/general/vpc-peering). If left unspecified, the job is not peered with any network.
+    dataflow_machine_type: The Dataflow machine type for evaluation components.
+    dataflow_disk_size_gb: The disk size (in GB) of the machine executing the evaluation run. If not set, defaulted to `50`.
+    dataflow_max_num_workers: The max number of workers executing the evaluation run. If not set, defaulted to `5`.
+    dataflow_service_account: Custom service account to run Dataflow jobs.
+    dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications
+    dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses.
+    encryption_spec_key_name:  Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created.
+    evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model.
+
+  Returns:
+    evaluation_metrics: ClassificationMetrics Artifact for LLM Text Classification.
+    evaluation_resource_name: If run on an user's managed VertexModel, the imported evaluation resource name. Empty if run on a publisher model.
+  """
+  # fmt: on
+  outputs = NamedTuple(
+      'outputs',
+      evaluation_metrics=ClassificationMetrics,
+      evaluation_resource_name=str,
+  )
+
+  get_vertex_model_task = dsl.importer(
+      artifact_uri=(
+          f'https://{location}-aiplatform.googleapis.com/v1/{model_name}'
+      ),
+      artifact_class=VertexModel,
+      metadata={'resourceName': model_name},
+  )
+  get_vertex_model_task.set_display_name('get-vertex-model')
+
+  eval_dataset_preprocessor_task = LLMEvaluationPreprocessorOp(
+      project=project,
+      location=location,
+      gcs_source_uris=batch_predict_gcs_source_uris,
+      input_field_name=input_field_name,
+      machine_type=machine_type,
+      service_account=service_account,
+      network=network,
+      encryption_spec_key_name=encryption_spec_key_name,
+  )
+  batch_predict_task = ModelBatchPredictOp(
+      project=project,
+      location=location,
+      model=get_vertex_model_task.outputs['artifact'],
+      job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}',
+      gcs_source_uris=eval_dataset_preprocessor_task.outputs[
+          'preprocessed_gcs_source_uris'
+      ],
+      instances_format=batch_predict_instances_format,
+      predictions_format=batch_predict_predictions_format,
+      gcs_destination_output_uri_prefix=batch_predict_gcs_destination_output_uri,
+      model_parameters=batch_predict_model_parameters,
+      encryption_spec_key_name=encryption_spec_key_name,
+  )
+
+  postprocessor_task = LLMEvaluationClassificationPredictionsPostprocessorOp(
+      project=project,
+      batch_prediction_results=batch_predict_task.outputs[
+          'gcs_output_directory'
+      ],
+      class_labels=evaluation_class_labels,
+      location=location,
+      machine_type=machine_type,
+      network=network,
+      service_account=service_account,
+      encryption_spec_key_name=encryption_spec_key_name,
+  )
+
+  eval_task = ModelEvaluationClassificationOp(
+      project=project,
+      location=location,
+      class_labels=postprocessor_task.outputs['postprocessed_class_labels'],
+      target_field_name=target_field_name,
+      predictions_gcs_source=postprocessor_task.outputs[
+          'postprocessed_predictions_gcs_source'
+      ],
+      prediction_label_column='prediction.classes',
+      prediction_score_column='prediction.scores',
+      predictions_format=batch_predict_predictions_format,
+      dataflow_machine_type=dataflow_machine_type,
+      dataflow_max_workers_num=dataflow_max_num_workers,
+      dataflow_disk_size_gb=dataflow_disk_size_gb,
+      dataflow_service_account=dataflow_service_account,
+      dataflow_subnetwork=dataflow_subnetwork,
+      dataflow_use_public_ips=dataflow_use_public_ips,
+      encryption_spec_key_name=encryption_spec_key_name,
+  )
+
+  import_evaluation_task = ModelImportEvaluationOp(
+      classification_metrics=eval_task.outputs['evaluation_metrics'],
+      model=get_vertex_model_task.outputs['artifact'],
+      dataset_type=batch_predict_instances_format,
+      dataset_paths=batch_predict_gcs_source_uris,
+      display_name=evaluation_display_name,
+  )
+
+  return outputs(
+      evaluation_metrics=eval_task.outputs['evaluation_metrics'],
+      evaluation_resource_name=import_evaluation_task.outputs[
+          'evaluation_resource_name'
+      ],
+  )

google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_text_generation_pipeline.py

@@ -0,0 +1,178 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Vertex Gen AI Evaluation for Text Generation/QA/Summarization tasks."""
+
+from typing import Dict, List, NamedTuple
+
+from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationPreprocessorOp
+from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationTextGenerationOp
+from google_cloud_pipeline_components._implementation.model_evaluation import ModelImportEvaluationOp
+from google_cloud_pipeline_components.types.artifact_types import VertexModel
+from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp
+from kfp import dsl
+# pylint: disable=unused-argument, unexpected-keyword-arg
+
+
+_PIPELINE_NAME = 'evaluation-llm-text-generation-pipeline'
+
+
+@dsl.pipeline(name=_PIPELINE_NAME)
+def evaluation_llm_text_generation_pipeline(  # pylint: disable=dangerous-default-value
+    project: str,
+    location: str,
+    batch_predict_gcs_source_uris: List[str],
+    batch_predict_gcs_destination_output_uri: str,
+    model_name: str = 'publishers/google/models/text-bison@002',
+    evaluation_task: str = 'text-generation',
+    input_field_name: str = 'input_text',
+    target_field_name: str = 'output_text',
+    batch_predict_instances_format: str = 'jsonl',
+    batch_predict_predictions_format: str = 'jsonl',
+    batch_predict_model_parameters: Dict[str, str] = {},
+    enable_row_based_metrics: bool = False,
+    machine_type: str = 'e2-highmem-16',
+    service_account: str = '',
+    network: str = '',
+    encryption_spec_key_name: str = '',
+    evaluation_display_name: str = 'evaluation-llm-text-generation-pipeline-{{$.pipeline_job_uuid}}',
+) -> NamedTuple(
+    'outputs', evaluation_metrics=dsl.Metrics, evaluation_resource_name=str
+):
+  # fmt: off
+  """LLM Text Generation Evaluation pipeline.
+
+  This pipeline supports evaluating large language models, publisher or managed
+  models, performing the following generative tasks: `summarization`, `question-answering`, and `text-generation`.
+
+  Args:
+    project: Required. The GCP project that runs the pipeline components.
+    location: Required. The GCP region that runs the pipeline components.
+    batch_predict_gcs_source_uris: Required. Google Cloud Storage URI(s) to your eval dataset instances data to run batch prediction on. The instances data should also contain the ground truth (target) data, used for evaluation. May contain wildcards. For more information on [wildcards](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). For more details about this [input config](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig). The content of gcs source files should be preset to one of the following formats:
+    1) Prediction & Evaluation Dataset format, guaranteeing "prompt" and "ground_truth" attributes are included
+    {
+      "prompt": "your input/prompt text",
+      "ground_truth": "your ground truth output text"
+    }
+    or
+    2) Tuning Dataset format, guaranteeing "input_text" and "output_text" attributes are included.
+    {
+      "input_text": "your input/prompt text",
+      "output_text": "your ground truth output text"
+    }
+    batch_predict_gcs_destination_output_uri: Required. The Google Cloud Storage location of the directory where the eval pipeline output is to be written to.
+    model_name: The Model name used to run evaluation. Must be a publisher Model or a managed Model sharing the same ancestor location. Starting this job has no impact on any existing deployments of the Model and their resources.
+    evaluation_task: The task that the large language model will be evaluated on. The evaluation component computes a set of metrics relevant to that specific task. Currently supported tasks are: `summarization`, `question-answering`, `text-generation`.
+    input_field_name: The field name of the input eval dataset instances that contains the input prompts to the LLM.
+    target_field_name: The field name of the eval dataset instance that contains an example reference text response. Alternatively referred to as the ground truth (or ground_truth_column) field. If not set, defaulted to `output_text`.
+    batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. Only "jsonl" is currently supported. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.
+    batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. Only "jsonl" is currently supported. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.
+    batch_predict_model_parameters: A map of parameters that govern the predictions. Some acceptable parameters include: maxOutputTokens, topK, topP, and temperature.
+    enable_row_based_metrics: Flag of if row based metrics is enabled, default value is false.
+    machine_type: The machine type of this custom job. If not set, defaulted to `e2-highmem-16`. More details: https://cloud.google.com/compute/docs/machine-resource
+    service_account: Sets the default service account for workload run-as account. The service account running the pipeline (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code Service Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project.
+    network: The full name of the Compute Engine network to which the job should be peered. For example, `projects/12345/global/networks/myVPC`. Format is of the form `projects/{project}/global/networks/{network}`. Where `{project}` is a project number, as in `12345`, and `{network}` is a network name, as in `myVPC`. To specify this field, you must have already configured VPC Network Peering for Vertex AI (https://cloud.google.com/vertex-ai/docs/general/vpc-peering). If left unspecified, the job is not peered with any network.
+    encryption_spec_key_name:  Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created.
+    evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model.
+
+  Returns:
+    evaluation_metrics: Metrics Artifact for LLM Text Generation.
+    evaluation_resource_name: If run on a user's managed VertexModel, the imported evaluation resource name. Empty if run on a publisher model.
+  """
+  # fmt: on
+  outputs = NamedTuple(
+      'outputs',
+      evaluation_metrics=dsl.Metrics,
+      evaluation_resource_name=str,
+  )
+
+  get_vertex_model_task = dsl.importer(
+      artifact_uri=(
+          f'https://{location}-aiplatform.googleapis.com/v1/{model_name}'
+      ),
+      artifact_class=VertexModel,
+      metadata={'resourceName': model_name},
+  )
+  get_vertex_model_task.set_display_name('get-vertex-model')
+
+  eval_dataset_preprocessor_task = LLMEvaluationPreprocessorOp(
+      project=project,
+      location=location,
+      gcs_source_uris=batch_predict_gcs_source_uris,
+      input_field_name=input_field_name,
+      machine_type=machine_type,
+      service_account=service_account,
+      network=network,
+      encryption_spec_key_name=encryption_spec_key_name,
+  )
+  batch_predict_task = ModelBatchPredictOp(
+      project=project,
+      location=location,
+      model=get_vertex_model_task.outputs['artifact'],
+      job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}',
+      gcs_source_uris=eval_dataset_preprocessor_task.outputs[
+          'preprocessed_gcs_source_uris'
+      ],
+      instances_format=batch_predict_instances_format,
+      predictions_format=batch_predict_predictions_format,
+      gcs_destination_output_uri_prefix=batch_predict_gcs_destination_output_uri,
+      model_parameters=batch_predict_model_parameters,
+      encryption_spec_key_name=encryption_spec_key_name,
+  )
+
+  eval_task = LLMEvaluationTextGenerationOp(
+      project=project,
+      location=location,
+      evaluation_task=evaluation_task,
+      target_field_name=f'instance.{target_field_name}',
+      predictions_format=batch_predict_predictions_format,
+      enable_row_based_metrics=enable_row_based_metrics,
+      joined_predictions_gcs_source=batch_predict_task.outputs[
+          'gcs_output_directory'
+      ],
+      machine_type=machine_type,
+      service_account=service_account,
+      network=network,
+      encryption_spec_key_name=encryption_spec_key_name,
+  )
+
+  with dsl.If(enable_row_based_metrics == True):
+    import_evaluation_task_with_row_based_metrics = ModelImportEvaluationOp(
+        metrics=eval_task.outputs['evaluation_metrics'],
+        row_based_metrics=eval_task.outputs['row_based_metrics'],
+        model=get_vertex_model_task.outputs['artifact'],
+        problem_type=evaluation_task,
+        dataset_type=batch_predict_predictions_format,
+        dataset_paths=batch_predict_gcs_source_uris,
+        display_name=evaluation_display_name,
+    )
+  with dsl.Else():
+    import_evaluation_task = ModelImportEvaluationOp(
+        metrics=eval_task.outputs['evaluation_metrics'],
+        model=get_vertex_model_task.outputs['artifact'],
+        problem_type=evaluation_task,
+        dataset_type=batch_predict_predictions_format,
+        dataset_paths=batch_predict_gcs_source_uris,
+        display_name=evaluation_display_name,
+    )
+
+  oneof = dsl.OneOf(
+      import_evaluation_task_with_row_based_metrics.outputs[
+          'evaluation_resource_name'
+      ],
+      import_evaluation_task.outputs['evaluation_resource_name'],
+  )
+  return outputs(
+      evaluation_metrics=eval_task.outputs['evaluation_metrics'],
+      evaluation_resource_name=oneof,
+  )

google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model based LLM evaluation preview components."""
+
+from google_cloud_pipeline_components.preview.model_evaluation.model_based_llm_evaluation.autosxs.autosxs_pipeline import autosxs_pipeline
+
+__all__ = [
+    'autosxs_pipeline',
+]

google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/autosxs_pipeline.py

@@ -0,0 +1,109 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Optimization AI Inference and AutoSxS pipeline function."""
+
+from typing import Any, Dict, List
+
+from google_cloud_pipeline_components import _placeholders
+from google_cloud_pipeline_components._implementation.llm import batch_prediction_pairwise
+from google_cloud_pipeline_components._implementation.llm import model_evaluation_text_generation_pairwise
+from google_cloud_pipeline_components._implementation.llm import online_evaluation_pairwise
+from kfp import dsl
+
+
+# pylint: disable=dangerous-default-value,g-bare-generic,unused-argument
+@dsl.pipeline(
+    name='autosxs-template',
+    description='Determines the SxS winrate between two models.',
+)
+def autosxs_pipeline(
+    evaluation_dataset: str,
+    task: str,
+    id_columns: List[str],
+    model_a: str = '',
+    model_b: str = '',
+    autorater_prompt_parameters: Dict[str, Dict[str, str]] = {},
+    model_a_prompt_parameters: Dict[str, Dict[str, str]] = {},
+    model_b_prompt_parameters: Dict[str, Dict[str, str]] = {},
+    response_column_a: str = '',
+    response_column_b: str = '',
+    model_a_parameters: Dict[str, str] = {},
+    model_b_parameters: Dict[str, str] = {},
+    human_preference_column: str = '',
+    project: str = _placeholders.PROJECT_ID_PLACEHOLDER,
+    location: str = _placeholders.LOCATION_PLACEHOLDER,
+    judgments_format: str = 'jsonl',
+    bigquery_destination_prefix: str = '',
+    experimental_args: Dict[str, Any] = {},
+):
+  # fmt: off
+  """Evaluates two models side-by-side using an arbiter model.
+
+  Args:
+    evaluation_dataset: A BigQuery table or comma-separated list of GCS paths to a JSONL dataset containing evaluation examples.
+    task: Evaluation task in the form `{task}@{version}`. task can be one of `[summarization, question_answering]`. Version is an integer with 3 digits or "latest". Ex: `summarization@001` or `question_answering@latest`.
+    id_columns: The columns which distinguish unique evaluation examples.
+    model_a: A fully-qualified model resource name (`projects/{project}/locations/{location}/models/{model}@{version}`) or publisher model resource name (`publishers/{publisher}/models/{model}`).  This parameter is optional if Model A responses are specified.
+    model_b: A fully-qualified model resource name (`projects/{project}/locations/{location}/models/{model}@{version}`) or publisher model resource name (`publishers/{publisher}/models/{model}`).  This parameter is optional if Model B responses are specified.
+    autorater_prompt_parameters: Map of autorater prompt parameters to columns or templates. The expected parameters are: `inference_instruction` (details on how to perform a task) and `inference_context` (content to reference to perform the task). As an example, `{'inference_context': {'column': 'my_prompt'}}` uses the evaluation dataset's `my_prompt` column for the AutoRater's context.
+    model_a_prompt_parameters: Map of Model A prompt template parameters to columns or templates. This parameter is optional if Model A predictions are predefined. Example - `{'prompt': {'column': 'my_prompt'}}` uses the evaluation dataset's `my_prompt` column for the prompt parameter named `prompt`.
+    model_b_prompt_parameters: Map of Model B prompt template parameters to columns or templates. This parameter is optional if Model B predictions are predefined. Example - `{'prompt': {'column': 'my_prompt'}}` uses the evaluation dataset's `my_prompt` column for the prompt parameter named `prompt`.
+    response_column_a: Either the name of a column in the evaluation dataset containing predefined predictions, or the name of the column in the Model A output containing predictions. If no value is provided, the correct model output column name will attempt to be inferred.
+    response_column_b: Either the name of a column in the evaluation dataset containing predefined predictions, or the name of the column in the Model B output containing predictions. If no value is provided, the correct model output column name will attempt to be inferred.
+    model_a_parameters: The parameters that govern the predictions from model A, such as temperature or maximum output tokens.
+    model_b_parameters: The parameters that govern the predictions from model B, such as temperature or maximum output tokens.
+    human_preference_column: The column containing ground truth winners for each example. Providing this parameter adds additional metrics for checking the AutoRater alignment with human preferences.
+    project: Project used to run custom jobs. This should be the same project used to run the pipeline.
+    location: Location used to run custom jobs. This should be the same location used to run the pipeline.
+    judgments_format: The format to write judgments to. Can be either `[json, bigquery]`.
+    bigquery_destination_prefix: BigQuery table to write judgments to if the specified format is 'bigquery'.
+    experimental_args: Experimentally released arguments. Subject to change.
+  """
+  # fmt: on
+  responses = batch_prediction_pairwise.batch_prediction_pairwise(
+      display_name='autosxs-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}',
+      evaluation_dataset=evaluation_dataset,
+      id_columns=id_columns,
+      task=task,
+      autorater_prompt_parameters=autorater_prompt_parameters,
+      response_column_a=response_column_a,
+      response_column_b=response_column_b,
+      model_a=model_a,
+      model_b=model_b,
+      model_a_prompt_parameters=model_a_prompt_parameters,
+      model_b_prompt_parameters=model_b_prompt_parameters,
+      model_a_parameters=model_a_parameters,
+      model_b_parameters=model_b_parameters,
+      human_preference_column=human_preference_column,
+      experimental_args=experimental_args,
+  ).set_display_name('AutoSxS Batch Prediction')
+
+  winners = online_evaluation_pairwise.online_evaluation_pairwise(
+      inference_output_uri=responses.outputs[
+          'preprocessed_evaluation_dataset_uri'
+      ],
+      id_columns=id_columns,
+      human_preference_column=human_preference_column,
+      task=task,
+      judgments_format=judgments_format,
+      bigquery_destination_prefix=bigquery_destination_prefix,
+      experimental_args=experimental_args,
+  ).set_display_name('AutoSxS Autorater')
+
+  model_evaluation_text_generation_pairwise.model_evaluation_text_generation_pairwise(
+      judgments_dir=winners.outputs['judgments_uri'],
+      human_preference_column=human_preference_column,
+  ).set_display_name(
+      'AutoSxS Metrics'
+  )

google_cloud_pipeline_components/proto/preflight_validations_pb2.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Protobuf Python Version: 0.20240110.0
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
+    b'\n\x13preflight_validations.proto\x12\x15preflight_validations"\x90\x02\n\x0eValidationItem\x12O\n\x0bsa_metadata\x18\x02'
+    b' \x01(\x0b\x32\x38.preflight_validations.GoogleCloudServiceAccountMetadataH\x00\x12P\n\x0equota_metadata\x18\x03'
+    b' \x01(\x0b\x32\x36.preflight_validations.GoogleCloudProjectQuotaMetadataH\x00\x12O\n\x0c\x61pi_metadata\x18\x04'
+    b' \x01(\x0b\x32\x37.preflight_validations.GoogleCloudApiEnablementMetadataH\x00\x42\n\n\x08metadata"\xeb\x01\n\x1fGoogleCloudProjectQuotaMetadata\x12\x14\n\x0cservice_name\x18\x01'
+    b' \x01(\t\x12s\n\x17metrics_recommendations\x18\x02'
+    b' \x03(\x0b\x32R.preflight_validations.GoogleCloudProjectQuotaMetadata.MetricsRecommendationsEntry\x1a=\n\x1bMetricsRecommendationsEntry\x12\x0b\n\x03key\x18\x01'
+    b' \x01(\t\x12\r\n\x05value\x18\x02'
+    b' \x01(\x03:\x02\x38\x01"P\n!GoogleCloudServiceAccountMetadata\x12\x16\n\x0eprincipal_name\x18\x01'
+    b' \x01(\t\x12\x13\n\x0bpermissions\x18\x02 \x03(\t"9\n'
+    b' GoogleCloudApiEnablementMetadata\x12\x15\n\rservice_names\x18\x01'
+    b' \x03(\tB\x02P\x01\x62\x06proto3'
+)
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(
+    DESCRIPTOR,
+    'google_cloud_pipeline_components.google_cloud_pipeline_components.proto.preflight_validations_pb2',
+    _globals,
+)
+if not _descriptor._USE_C_DESCRIPTORS:
+  _globals['DESCRIPTOR']._loaded_options = None
+  _globals['DESCRIPTOR']._serialized_options = b'P\001'
+  _globals[
+      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
+  ]._loaded_options = None
+  _globals[
+      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
+  ]._serialized_options = b'8\001'
+  _globals['_VALIDATIONITEM']._serialized_start = 142
+  _globals['_VALIDATIONITEM']._serialized_end = 414
+  _globals['_GOOGLECLOUDPROJECTQUOTAMETADATA']._serialized_start = 417
+  _globals['_GOOGLECLOUDPROJECTQUOTAMETADATA']._serialized_end = 652
+  _globals[
+      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
+  ]._serialized_start = 591
+  _globals[
+      '_GOOGLECLOUDPROJECTQUOTAMETADATA_METRICSRECOMMENDATIONSENTRY'
+  ]._serialized_end = 652
+  _globals['_GOOGLECLOUDSERVICEACCOUNTMETADATA']._serialized_start = 654
+  _globals['_GOOGLECLOUDSERVICEACCOUNTMETADATA']._serialized_end = 734
+  _globals['_GOOGLECLOUDAPIENABLEMENTMETADATA']._serialized_start = 736
+  _globals['_GOOGLECLOUDAPIENABLEMENTMETADATA']._serialized_end = 793
+# @@protoc_insertion_point(module_scope)

google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py

@@ -151,8 +151,9 @@ def evaluation_llm_text_generation_pipeline(  # pylint: disable=dangerous-defaul
   eval_task = LLMEvaluationTextGenerationOp(
       project=project,
       location=location,
+      model_name=model_name,
       evaluation_task=evaluation_task,
-      target_field_name=f'instance.{target_field_name}',
+      target_field_name=target_field_name,
       predictions_format=batch_predict_predictions_format,
       enable_row_based_metrics=enable_row_based_metrics,
       joined_predictions_gcs_source=batch_predict_task.outputs[

google_cloud_pipeline_components/version.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Google Cloud Pipeline Components version."""
 
-__version__ = "2.16.1"
+__version__ = "2.17.0"

{google_cloud_pipeline_components-2.16.1.dist-info → google_cloud_pipeline_components-2.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-cloud-pipeline-components
-Version: 2.16.1
+Version: 2.17.0
 Summary: This SDK enables a set of First Party (Google owned) pipeline components that allow users to take their experience from Vertex AI SDK and other Google Cloud services and create a corresponding pipeline using KFP or Managed Pipelines.
 Home-page: https://github.com/kubeflow/pipelines/tree/master/components/google-cloud
 Author: The Google Cloud Pipeline Components authors
@@ -10,6 +10,7 @@ Project-URL: User Documentation, https://cloud.google.com/vertex-ai/docs/pipelin
 Project-URL: Reference Documentation, https://google-cloud-pipeline-components.readthedocs.io/
 Project-URL: Source, https://github.com/kubeflow/pipelines/tree/master/components/google-cloud
 Project-URL: Release Notes, https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/RELEASE.md
+Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Operating System :: Unix
 Classifier: Operating System :: MacOS
@@ -30,24 +31,24 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8.0,<3.12.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: google-api-core !=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5
-Requires-Dist: kfp <=2.7.0,>=2.6.0
-Requires-Dist: google-cloud-aiplatform <2,>=1.14.0
-Requires-Dist: Jinja2 <4,>=3.1.2
+Requires-Dist: google-api-core (!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5)
+Requires-Dist: kfp (<=2.7.0,>=2.6.0)
+Requires-Dist: google-cloud-aiplatform (<2,>=1.14.0)
+Requires-Dist: Jinja2 (<4,>=3.1.2)
 Provides-Extra: docs
-Requires-Dist: protobuf <5,>=4.21.1 ; extra == 'docs'
-Requires-Dist: grpcio-status <=1.47.0 ; extra == 'docs'
-Requires-Dist: commonmark ==0.9.1 ; extra == 'docs'
-Requires-Dist: autodocsumm ==0.2.9 ; extra == 'docs'
-Requires-Dist: sphinx <6.0.0,>=5.0.2 ; extra == 'docs'
-Requires-Dist: sphinx-immaterial ==0.9.0 ; extra == 'docs'
-Requires-Dist: sphinx-rtd-theme ==2.0.0 ; extra == 'docs'
-Requires-Dist: m2r2 ==0.3.3.post2 ; extra == 'docs'
-Requires-Dist: sphinx-notfound-page ==0.8.3 ; extra == 'docs'
+Requires-Dist: protobuf (<5,>=4.21.1) ; extra == 'docs'
+Requires-Dist: grpcio-status (<=1.47.0) ; extra == 'docs'
+Requires-Dist: commonmark (==0.9.1) ; extra == 'docs'
+Requires-Dist: autodocsumm (==0.2.9) ; extra == 'docs'
+Requires-Dist: sphinx (<6.0.0,>=5.0.2) ; extra == 'docs'
+Requires-Dist: sphinx-immaterial (==0.9.0) ; extra == 'docs'
+Requires-Dist: sphinx-rtd-theme (==2.0.0) ; extra == 'docs'
+Requires-Dist: m2r2 (==0.3.3.post2) ; extra == 'docs'
+Requires-Dist: sphinx-notfound-page (==0.8.3) ; extra == 'docs'
 Provides-Extra: tests
-Requires-Dist: mock >=4.0.0 ; extra == 'tests'
-Requires-Dist: flake8 >=3.0.0 ; extra == 'tests'
-Requires-Dist: pytest >=6.0.0 ; extra == 'tests'
+Requires-Dist: mock (>=4.0.0) ; extra == 'tests'
+Requires-Dist: flake8 (>=3.0.0) ; extra == 'tests'
+Requires-Dist: pytest (>=6.0.0) ; extra == 'tests'
 
 # Google Cloud Pipeline Components
 
@@ -92,3 +93,5 @@ Use the following command to install Google Cloud Pipeline Components from [PyPI
 ```shell
 pip install -U google-cloud-pipeline-components
 ```
+
+

{google_cloud_pipeline_components-2.16.1.dist-info → google_cloud_pipeline_components-2.17.0.dist-info}/RECORD

@@ -2,7 +2,7 @@ google_cloud_pipeline_components/__init__.py,sha256=HOU1WNUeWpY1HjluW8vbI7Cai_EF
 google_cloud_pipeline_components/_image.py,sha256=lANDYNk1WSuGZSoTTRcWdjsUvCkkA-PmwouTM9Et7fY,828
 google_cloud_pipeline_components/_placeholders.py,sha256=9yIbieYjZkZJg4xuKklxWQVn3Z-9ryb9Q6V15Jqpe98,3090
 google_cloud_pipeline_components/utils.py,sha256=9FG7umyEXhyUvtNeC46NuQ04olDMR3o-Wp78V1xs8GY,11045
-google_cloud_pipeline_components/version.py,sha256=SP3sOlCOMI-zlRMGGvuqYoBYWVLHF-u5h5tqKoQiyMM,678
+google_cloud_pipeline_components/version.py,sha256=oiNOAYZSDGBwtvZJCPchicTrMKCZ_ty9jdtzziPGfgY,678
 google_cloud_pipeline_components/_implementation/__init__.py,sha256=sb6SfJl6rt3AKjiWxd-KO9DSiZ3PzGZRcsqKuc1A2Cg,606
 google_cloud_pipeline_components/_implementation/llm/__init__.py,sha256=sb6SfJl6rt3AKjiWxd-KO9DSiZ3PzGZRcsqKuc1A2Cg,606
 google_cloud_pipeline_components/_implementation/llm/batch_prediction_pairwise.py,sha256=mARB-tDYFr0tpBrLCIh481H4LDuXdr_8UyKVUaOF5Cw,7569
@@ -30,13 +30,13 @@ google_cloud_pipeline_components/_implementation/llm/utils.py,sha256=E250cmvw0QU
 google_cloud_pipeline_components/_implementation/llm/utils_test.py,sha256=co8gWyrowY5CpkFNsaLGQlD_gpIykkVI7czxIizp5cM,2864
 google_cloud_pipeline_components/_implementation/llm/validate_pipeline.py,sha256=35E2jx3wG_1-qnF9u6GygEmekJ3dG9LklJx2Bh6ayJ8,4350
 google_cloud_pipeline_components/_implementation/llm/generated/__init__.py,sha256=LYxMCPeZcfGqRbt3mo3hY7U02A6G8aWdP_RtdoqxNrQ,606
-google_cloud_pipeline_components/_implementation/llm/generated/refined_image_versions.py,sha256=dSLCXSQKd4M1HxSIhoAmASPIqDa2qx7BMzHB97vIEz8,763
+google_cloud_pipeline_components/_implementation/llm/generated/refined_image_versions.py,sha256=UJ2J8TOz1rDNgSqUe-Mj2k0-G_Eb8fu7Yg4BwaG2IrM,763
 google_cloud_pipeline_components/_implementation/model/__init__.py,sha256=KmOW74re0WZ93DWM1lqqQYbv6w1aIW66BMV3gaAdg3s,811
 google_cloud_pipeline_components/_implementation/model/get_model/__init__.py,sha256=cXMkDUZHVSbXeXSa3qsI6Ef8Tad9nmusw5NUZaYORdE,662
 google_cloud_pipeline_components/_implementation/model/get_model/component.py,sha256=H2sbMTWCw8nMDMT-Ni9-pdzVXEFmHYjtP3z1LcI5m5w,2307
 google_cloud_pipeline_components/_implementation/model_evaluation/__init__.py,sha256=oB1e-LSzK0pctcXf6M9Uiu3_83QxC7RnUy4lU1dlOJs,4692
 google_cloud_pipeline_components/_implementation/model_evaluation/utils.py,sha256=9V34RtPZSRNeBwcsImaZM6YC3T7CafT_E00Iby4KHxw,3540
-google_cloud_pipeline_components/_implementation/model_evaluation/version.py,sha256=QHCvg6WZkQq-VdX7hbWCQHK-7SjHpxSppGa9ppyJNmk,963
+google_cloud_pipeline_components/_implementation/model_evaluation/version.py,sha256=XvzrHsB6WzGMjkz8-PvSJhQToWTbUByPPSoz9DLzKF4,963
 google_cloud_pipeline_components/_implementation/model_evaluation/chunking/__init__.py,sha256=PRHVsIq1dFb0mweuU0kfUUP90FbX--kxdBGCpwfLTgA,665
 google_cloud_pipeline_components/_implementation/model_evaluation/chunking/component.py,sha256=lv0eNdIBtnCUs9wi9CGUWE_fd4zodGN7M8fTBiCg0g0,4436
 google_cloud_pipeline_components/_implementation/model_evaluation/chunking/feature_store_grounding_pipeline.py,sha256=zGCMJhBCWBV-Dg5gz-wpheLQhD-GuEW_fgcb0cWa78U,4167
@@ -57,6 +57,8 @@ google_cloud_pipeline_components/_implementation/model_evaluation/feature_extrac
 google_cloud_pipeline_components/_implementation/model_evaluation/feature_extractor/component.py,sha256=u4P3hSGBnFXtn8MKb8g1J9FAA74VpowBmPTmSDFq9J4,5539
 google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluated_annotation/__init__.py,sha256=-znXeLR3Uhon4BsRcAqHdVhcSUCqkY7rFouQFLD6Atg,684
 google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluated_annotation/component.py,sha256=d1QlNA_hpSJNgvwRJFGKZnVi0zc0KhZEHzVkq5wJWXQ,3053
+google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/__init__.py,sha256=eG4D5YbXHkveV6PP1uX8ZL48LCZ1mUtKGKb520dpILg,674
+google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/component.py,sha256=uZEtGhRtkCtjJFxBHFv8LSm_m7KlJxgmMhc2Whw6Mrw,8026
 google_cloud_pipeline_components/_implementation/model_evaluation/llm_classification_postprocessor/__init__.py,sha256=kEQ4aaKnV-KulHqbhb6eJIZzk4O7uSBRPzm_e3q_hcA,697
 google_cloud_pipeline_components/_implementation/model_evaluation/llm_classification_postprocessor/component.py,sha256=H0g7nMK3JVdQonLe5jpKvvzm9N2YzFGQUWJwwfxQ33s,11227
 google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding/__init__.py,sha256=CUSeP0l2KFuo2wbw00DE5Zur0jpgHpZ1aThg7yqWuGY,680
@@ -64,9 +66,9 @@ google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding/
 google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding_retrieval/__init__.py,sha256=Q9YimgEKkKP8QW8fV50nNRjWXdt0_90Qr_gfQ0A9fao,691
 google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding_retrieval/component.py,sha256=er26AxV0ydpKFbhrLs2IIia9hbLxXhe78Q-fQvvK_20,7265
 google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/__init__.py,sha256=gDyltb_vTZRncaVZbMUkXYBHZsEg_CuaPAQVWUOGy7c,671
-google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py,sha256=rclGlXxDr6vozQ6elDDCDbdJXWjG461Z9nmTaiKGsFs,7478
+google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py,sha256=vU3EsXoquUpsbu999rByKkZTvMYLTkH-Xdj2ycI-KnA,7598
 google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/__init__.py,sha256=LLvQQ9Mv_md1POK_XErR7NUr-XAZX28w5KD0EQiT32w,684
-google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/component.py,sha256=L6dPaWuOr-BhP7yAatFGVTX2MYTuI2f88W_oR5ee88E,9170
+google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/component.py,sha256=HOj4_2DOXVP0s0NkVAY1_q4j4okqEcHpdfPsE7W2dI8,9131
 google_cloud_pipeline_components/_implementation/model_evaluation/llm_information_retrieval_preprocessor/__init__.py,sha256=UcAfohvqwJ7X8rlV6I2RsZ5ohYWbafXI5yYRAAwvspE,695
 google_cloud_pipeline_components/_implementation/model_evaluation/llm_information_retrieval_preprocessor/component.py,sha256=hpwra0rXUP_LG6WNdnB9RD0cy1Gn3InrzAUJIk-oRc4,8142
 google_cloud_pipeline_components/_implementation/model_evaluation/llm_retrieval_metrics/__init__.py,sha256=oVVVTCE230KYGcuz_rXYEuypZmm6OKg7K2BsFtpcDw0,701
@@ -360,15 +362,21 @@ google_cloud_pipeline_components/preview/llm/rlhf/__init__.py,sha256=sb6SfJl6rt3
 google_cloud_pipeline_components/preview/llm/rlhf/component.py,sha256=9XpSVUga81GTqkn4qMBddhc41IzY0-p4IBC4UFP50G4,15339
 google_cloud_pipeline_components/preview/model_evaluation/__init__.py,sha256=n076N7S2HEdMuSZ8cyKM6acjCjslOHjCx4SR49asRME,2121
 google_cloud_pipeline_components/preview/model_evaluation/data_bias_component.py,sha256=YiwkWfbGymX_lDIg_x7AP6nYMm3MQp_NgV8xuSZxCpU,5791
+google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_classification_pipeline.py,sha256=IjCIakZFh7KB6kmku7ztaH-qFm9LoEctwwMtxmDOg5g,11262
+google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_text_generation_pipeline.py,sha256=afS9-rkKbQHDwOldtFC8w7Xv9UOqIU-Xt6ZEim14fqg,10761
 google_cloud_pipeline_components/preview/model_evaluation/feature_attribution_component.py,sha256=XWrI1inQ9hKixFrp2LUdgu7hONYUvbsxv2GXZ-UTkCY,7450
 google_cloud_pipeline_components/preview/model_evaluation/feature_attribution_graph_component.py,sha256=jesgBUKbIB_qQoYb5-Bv_LBbFHl0tPyMlVFx-o1eE6k,13624
 google_cloud_pipeline_components/preview/model_evaluation/model_bias_component.py,sha256=R8WhT8jf_OOpMuABRh2BYTDEcfiGAf6VA-vFgiTymYY,6674
 google_cloud_pipeline_components/preview/model_evaluation/model_evaluation_import_component.py,sha256=ePHbJqWYbZR5wCgJ8aVgDrOcWSoAiVq7df6HxTw1gdY,8118
 google_cloud_pipeline_components/preview/model_evaluation/utils.py,sha256=oRlEvA3zMSTzgxJklZD0A-BzFDx0-PsBHBXZ4kmaREY,7539
+google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/__init__.py,sha256=ee_EyGhwqXIjR3Rx9t-o2gV9TssU-VErMU7LtDA7s9k,838
+google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/__init__.py,sha256=sb6SfJl6rt3AKjiWxd-KO9DSiZ3PzGZRcsqKuc1A2Cg,606
+google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/autosxs_pipeline.py,sha256=rymTHXZZCMFtwzf9qOjDx2qd2yCdTYzUO0kfja7kAiE,7055
 google_cloud_pipeline_components/preview/starry_net/__init__.py,sha256=yEpcicPBsGqgahrCtcJ06a_7E2pUgdp32U0yWwFRy7E,796
 google_cloud_pipeline_components/preview/starry_net/component.py,sha256=SdmUfL7xZfK8yWOuVqDczqTqJMShAga3rJxgj4mFCxE,24424
 google_cloud_pipeline_components/proto/__init__.py,sha256=aiPUc6gpQwG9cRTYfw3ChFCJfDr3vAIsm2eMYUDJjJQ,661
 google_cloud_pipeline_components/proto/gcp_resources_pb2.py,sha256=ssNNm4zjiWbuBUS7IH6kyrvvfmcC_Z5F7hOAuQe_YLk,2134
+google_cloud_pipeline_components/proto/preflight_validations_pb2.py,sha256=ZZFTISXvqQl-284O_zPJrgXO4REPLwn3qxXg8BTJHM0,3056
 google_cloud_pipeline_components/proto/task_error_pb2.py,sha256=PPz4luEm3Pa3YDn4mtpqEPl8QVSFa44pV8W2KbWRq1I,1235
 google_cloud_pipeline_components/proto/template_metadata_pb2.py,sha256=jAIJgc5UWYz8E0Vf3IbjPlyiWI3kCNYrJHyAUaQTpgc,10067
 google_cloud_pipeline_components/types/__init__.py,sha256=1WFkL49QEy-gNb6ywQOE4yZkD7DoULAeiL1tLdb3S28,606
@@ -544,7 +552,7 @@ google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_tabular_p
 google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_unstructure_data_pipeline.py,sha256=LlMSMVbiZGXOZo4SqEQZZu9GQWWzHnwhrygwmDkqVdA,42457
 google_cloud_pipeline_components/v1/model_evaluation/evaluation_feature_attribution_pipeline.py,sha256=wEu67EPXb9x9PcYMYmnuETKbAz_lP5Kewv439IiFxIM,51226
 google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_classification_pipeline.py,sha256=vF_EOPg7S3UCTCNw6GrhQz_Sn8ctPZ-JpJEt6JlSF3Y,12036
-google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py,sha256=pqRLTWQpJLwn4kKQ5o5hx9_FlcPDcIC0Z3joykJHBhs,11854
+google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py,sha256=1WD_fbYSdYrXMUEJiULgMY5Vi2BlklmvwkhtZWZeTxs,11869
 google_cloud_pipeline_components/v1/model_evaluation/forecasting_component.py,sha256=gOnvKAJWa3velczeuVBCzW6b_tcc2v_lNFqHXGhjD44,10017
 google_cloud_pipeline_components/v1/model_evaluation/regression_component.py,sha256=eFrjrKQot3-SlRCoKoTOEsyp2Xj0GfDtrjpxTDKAHYY,9117
 google_cloud_pipeline_components/v1/model_evaluation/model_based_llm_evaluation/__init__.py,sha256=57gYyoAJwLIeXoLTTyVqOWgkj7dlUP4IKM4OmMabCRM,828
@@ -554,8 +562,8 @@ google_cloud_pipeline_components/v1/vertex_notification_email/__init__.py,sha256
 google_cloud_pipeline_components/v1/vertex_notification_email/component.py,sha256=Dau8ZI0mzLBnLOUBQm6EtK8gbtX1u57t76Ud5qlg9xc,2163
 google_cloud_pipeline_components/v1/wait_gcp_resources/__init__.py,sha256=w6dfz-rYsYnxFapRH1Dix3GVz0mhPW0m1IVpE6z8jbg,878
 google_cloud_pipeline_components/v1/wait_gcp_resources/component.py,sha256=Nsfj5c3eeZq83fHLvv2IlpK4jrjxLxRksFYOl5W6JnA,2468
-google_cloud_pipeline_components-2.16.1.dist-info/LICENSE,sha256=VAc1R5OxOELKsX5L5Ldp5THfNtxtt1cMIZBaC0Jdj5Q,13118
-google_cloud_pipeline_components-2.16.1.dist-info/METADATA,sha256=mKcwBgD0FKhbdCC315q_jU7a2GijYVZ_UtPRQHbqfMQ,5810
-google_cloud_pipeline_components-2.16.1.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-google_cloud_pipeline_components-2.16.1.dist-info/top_level.txt,sha256=E8T4T8KGMGLXbHvt2goa98oezRpxryPC6QhWBZ27Hhc,33
-google_cloud_pipeline_components-2.16.1.dist-info/RECORD,,
+google_cloud_pipeline_components-2.17.0.dist-info/LICENSE,sha256=VAc1R5OxOELKsX5L5Ldp5THfNtxtt1cMIZBaC0Jdj5Q,13118
+google_cloud_pipeline_components-2.17.0.dist-info/METADATA,sha256=X7lKaGEXx4JV6NTMgGp9sZjOB8fHRl25ULQln2Ov6tM,5862
+google_cloud_pipeline_components-2.17.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+google_cloud_pipeline_components-2.17.0.dist-info/top_level.txt,sha256=E8T4T8KGMGLXbHvt2goa98oezRpxryPC6QhWBZ27Hhc,33
+google_cloud_pipeline_components-2.17.0.dist-info/RECORD,,

{google_cloud_pipeline_components-2.16.1.dist-info → google_cloud_pipeline_components-2.17.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.2.0)
+Generator: bdist_wheel (0.37.0)
 Root-Is-Purelib: true
 Tag: py3-none-any