google-cloud-pipeline-components 2.6.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

google_cloud_pipeline_components/container/v1/automl_training_job/image/remote_runner.py

@@ -0,0 +1,250 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""GCP remote runner for AutoML image training pipelines based on the AI Platform SDK."""
+
+import logging
+from typing import Any, Dict, Optional, Sequence
+
+from google.api_core import retry
+from google.cloud.aiplatform import datasets
+from google.cloud.aiplatform import gapic
+from google.cloud.aiplatform import initializer
+from google.cloud.aiplatform import models
+from google.cloud.aiplatform import schema
+from google.cloud.aiplatform import training_jobs
+from google.cloud.aiplatform_v1.types import model
+from google.cloud.aiplatform_v1.types import training_pipeline
+from google_cloud_pipeline_components.container.v1.gcp_launcher import pipeline_remote_runner
+from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import error_util
+
+from google.protobuf import struct_pb2
+from google.protobuf import json_format
+
+
+_GET_PIPELINE_RETRY_DEADLINE_SECONDS = 10.0 * 60.0
+_CLASSIFICATION = 'classification'
+_OBJECT_DETECTION = 'object_detection'
+
+
+# pylint: disable=protected-access
+def create_payload(
+    project: str,
+    location: str,
+    display_name: Optional[str] = None,
+    prediction_type: str = _CLASSIFICATION,
+    multi_label: bool = False,
+    model_type: str = 'CLOUD',
+    labels: Optional[Dict[str, str]] = None,
+    dataset: Optional[str] = None,
+    disable_early_stopping: bool = False,
+    training_encryption_spec_key_name: Optional[str] = None,
+    model_encryption_spec_key_name: Optional[str] = None,
+    model_display_name: Optional[str] = None,
+    training_fraction_split: Optional[float] = None,
+    validation_fraction_split: Optional[float] = None,
+    test_fraction_split: Optional[float] = None,
+    budget_milli_node_hours: Optional[int] = None,
+    training_filter_split: Optional[str] = None,
+    validation_filter_split: Optional[str] = None,
+    test_filter_split: Optional[str] = None,
+    base_model: Optional[str] = None,
+    incremental_train_base_model: Optional[str] = None,
+    parent_model: Optional[str] = None,
+    is_default_version: Optional[bool] = True,
+    model_version_aliases: Optional[Sequence[str]] = None,
+    model_version_description: Optional[str] = None,
+    model_labels: Optional[Dict[str, str]] = None,
+) -> str:
+  """Creates a AutoML Image Training Job payload."""
+  # Override default model_type for object_detection
+  if model_type == 'CLOUD' and prediction_type == _OBJECT_DETECTION:
+    model_type = 'CLOUD_HIGH_ACCURACY_1'
+
+  training_encryption_spec = initializer.global_config.get_encryption_spec(
+      encryption_spec_key_name=training_encryption_spec_key_name
+  )
+  model_encryption_spec = initializer.global_config.get_encryption_spec(
+      encryption_spec_key_name=model_encryption_spec_key_name
+  )
+
+  # Training task inputs.
+  training_task_inputs = {
+      # required inputs
+      'modelType': model_type,
+      'budgetMilliNodeHours': budget_milli_node_hours,
+      # optional inputs
+      'disableEarlyStopping': disable_early_stopping,
+  }
+  if prediction_type == _CLASSIFICATION:
+    training_task_inputs['multiLabel'] = multi_label
+  if incremental_train_base_model:
+    training_task_inputs['uptrainBaseModelId'] = incremental_train_base_model
+
+  training_task_definition = getattr(
+      schema.training_job.definition, f'automl_image_{prediction_type}'
+  )
+
+  # Input data config.
+  input_data_config = training_jobs._TrainingJob._create_input_data_config(
+      dataset=dataset and datasets.ImageDataset(dataset_name=dataset),
+      training_fraction_split=training_fraction_split,
+      validation_fraction_split=validation_fraction_split,
+      test_fraction_split=test_fraction_split,
+      training_filter_split=training_filter_split,
+      validation_filter_split=validation_filter_split,
+      test_filter_split=test_filter_split,
+  )
+
+  # Model to upload.
+  model_to_upload = model.Model(
+      display_name=model_display_name or display_name,
+      labels=model_labels or labels,
+      encryption_spec=model_encryption_spec,
+      version_aliases=models.ModelRegistry._get_true_alias_list(
+          model_version_aliases, is_default_version
+      ),
+      version_description=model_version_description,
+  )
+
+  # Sets base_model.
+  if base_model:
+    training_task_inputs['baseModelId'] = base_model
+
+  # Create training task inputs.
+  training_task_inputs_struct = struct_pb2.Struct()
+  training_task_inputs_struct.update(training_task_inputs)
+
+  # Gets parent_model.
+  parent_model = models.ModelRegistry._get_true_version_parent(
+      parent_model=parent_model,
+      project=project,
+      location=location,
+  )
+
+  pipeline = training_pipeline.TrainingPipeline(
+      display_name=display_name,
+      training_task_definition=training_task_definition,
+      training_task_inputs=struct_pb2.Value(
+          struct_value=training_task_inputs_struct
+      ),
+      model_to_upload=model_to_upload,
+      parent_model=parent_model,
+      input_data_config=input_data_config,
+      labels=labels,
+      encryption_spec=training_encryption_spec,
+  )
+
+  return json_format.MessageToJson(
+      pipeline._pb, preserving_proto_field_name=True
+  )
+
+
+# pylint: enable=protected-access
+
+
+def create_pipeline_with_client(
+    pipeline_client: gapic.PipelineServiceClient,
+    parent,
+    pipeline_spec: Any,
+):
+  """Creates a training pipeline with the client."""
+  created_pipeline = None
+  try:
+    logging.info(
+        'Creating AutoML Vision training pipeline with sanitized pipeline'
+        ' spec: %s',
+        pipeline_spec,
+    )
+    created_pipeline = pipeline_client.create_training_pipeline(
+        parent=parent,
+        training_pipeline=training_pipeline.TrainingPipeline(**pipeline_spec),
+    )
+  except (ConnectionError, RuntimeError) as err:
+    error_util.exit_with_internal_error(err.args[0])
+  return created_pipeline
+
+
+def get_pipeline_with_client(
+    pipeline_client: gapic.PipelineServiceClient, pipeline_name: str
+):
+  """Gets training pipeline state with the client."""
+  get_automl_vision_training_pipeline = None
+  try:
+    get_automl_vision_training_pipeline = pipeline_client.get_training_pipeline(
+        name=pipeline_name,
+        retry=retry.Retry(deadline=_GET_PIPELINE_RETRY_DEADLINE_SECONDS),
+    )
+  except (ConnectionError, RuntimeError) as err:
+    error_util.exit_with_internal_error(err.args[0])
+  return get_automl_vision_training_pipeline
+
+
+def create_pipeline(
+    type: str,  # pylint: disable=redefined-builtin
+    project: str,
+    location: str,
+    gcp_resources: str,
+    **kwargs: Dict[str, Any],
+):
+  """Create and poll AutoML Vision training pipeline status till it reaches a final state.
+
+  This follows the typical launching logic:
+  1. Read if the training pipeline already exists in gcp_resources
+     - If already exists, jump to step 3 and poll the pipeline status. This
+     happens
+     if the launcher container experienced unexpected termination, such as
+     preemption
+  2. Deserialize the payload into the pipeline spec and create the training
+  pipeline
+  3. Poll the training pipeline status every
+  pipeline_remote_runner._POLLING_INTERVAL_IN_SECONDS seconds
+     - If the training pipeline is succeeded, return succeeded
+     - If the training pipeline is cancelled/paused, it's an unexpected
+     scenario so return failed
+     - If the training pipeline is running, continue polling the status
+
+  Also retry on ConnectionError up to
+  pipeline_remote_runner._CONNECTION_ERROR_RETRY_LIMIT times during the poll.
+
+  Args:
+    type: Job type.
+    project: Project name.
+    location: Location to start the training job.
+    gcp_resources: URI for storing GCP resources.
+    **kwargs: Extra args for creating the payload.
+  """
+  remote_runner = pipeline_remote_runner.PipelineRemoteRunner(
+      type, project, location, gcp_resources
+  )
+
+  try:
+    # Create AutoML vision training pipeline if it does not exist
+    pipeline_name = remote_runner.check_if_pipeline_exists()
+    if pipeline_name is None:
+      payload = create_payload(project, location, **kwargs)
+      logging.info(
+          'AutoML Vision training payload formatted: %s',
+          payload,
+      )
+      pipeline_name = remote_runner.create_pipeline(
+          create_pipeline_with_client,
+          payload,
+      )
+
+    # Poll AutoML Vision training pipeline status until
+    # "PipelineState.PIPELINE_STATE_SUCCEEDED"
+    remote_runner.poll_pipeline(get_pipeline_with_client, pipeline_name)
+
+  except (ConnectionError, RuntimeError) as err:
+    error_util.exit_with_internal_error(err.args[0])

google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_text_generation_pipeline.py

@@ -40,6 +40,7 @@ def evaluation_llm_text_generation_pipeline(  # pylint: disable=dangerous-defaul
     batch_predict_instances_format: str = 'jsonl',
     batch_predict_predictions_format: str = 'jsonl',
     batch_predict_model_parameters: Dict[str, str] = {},
+    enable_row_based_metrics: bool = False,
     machine_type: str = 'e2-highmem-16',
     service_account: str = '',
     network: str = '',
@@ -74,7 +75,7 @@ def evaluation_llm_text_generation_pipeline(  # pylint: disable=dangerous-defaul
 
   Returns:
     evaluation_metrics: Metrics Artifact for LLM Text Generation.
-    evaluation_resource_name: If run on an user's managed VertexModel, the imported evaluation resource name. Empty if run on a publisher model.
+    evaluation_resource_name: If run on a user's managed VertexModel, the imported evaluation resource name. Empty if run on a publisher model.
   """
   # fmt: on
   outputs = NamedTuple(
@@ -123,6 +124,7 @@ def evaluation_llm_text_generation_pipeline(  # pylint: disable=dangerous-defaul
       evaluation_task=evaluation_task,
       target_field_name=f'instance.{target_field_name}',
       predictions_format=batch_predict_predictions_format,
+      enable_row_based_metrics=enable_row_based_metrics,
       joined_predictions_gcs_source=batch_predict_task.outputs[
           'gcs_output_directory'
       ],
@@ -134,6 +136,9 @@ def evaluation_llm_text_generation_pipeline(  # pylint: disable=dangerous-defaul
 
   import_evaluation_task = ModelImportEvaluationOp(
       metrics=eval_task.outputs['evaluation_metrics'],
+      row_based_metrics=eval_task.outputs['row_based_metrics']
+      if enable_row_based_metrics
+      else None,
       model=get_vertex_model_task.outputs['artifact'],
       problem_type=evaluation_task,
       dataset_type=batch_predict_predictions_format,

google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model based LLM evaluation preview components."""
+
+from google_cloud_pipeline_components.preview.model_evaluation.model_based_llm_evaluation.autosxs.autosxs_pipeline import autosxs_pipeline
+
+__all__ = [
+    'autosxs_pipeline',
+]

google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/autosxs_pipeline.py

@@ -0,0 +1,234 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Optimization AI Inference and AutoSxS pipeline function."""
+
+from typing import Any, Dict, List
+
+from google_cloud_pipeline_components import _placeholders
+from google_cloud_pipeline_components._implementation.llm import arbiter_preprocess
+from google_cloud_pipeline_components._implementation.llm import autosxs_arbiter
+from google_cloud_pipeline_components._implementation.llm import autosxs_metrics_computer
+from google_cloud_pipeline_components._implementation.llm import function_based
+from google_cloud_pipeline_components._implementation.llm import task_preprocess
+from google_cloud_pipeline_components.types import artifact_types
+from google_cloud_pipeline_components.v1 import batch_predict_job
+from kfp import dsl
+
+
+# pylint: disable=no-value-for-parameter
+@dsl.pipeline(
+    name='predictions-pipeline',
+    description='Runs the prediction pipeline for one of the two SxS models.',
+)
+def _get_predictions(
+    name: str,
+    project: str,
+    location: str,
+    model: str,
+    model_parameters: Dict[str, str],
+    prediction_inputs: List[str],
+    is_model_inference: bool,
+) -> str:
+  """Makes predictions for a given model."""
+  with dsl.If(is_model_inference == True, name='Inference Required'):  # pylint: disable=singleton-comparison
+    get_vertex_model_task = dsl.importer(
+        artifact_uri=(
+            f'https://{location}-aiplatform.googleapis.com/v1/{model}'
+        ),
+        artifact_class=artifact_types.VertexModel,
+        metadata={'resourceName': model},
+    ).set_display_name('Import Vertex Model Artifact')
+
+    batch_predict_task = batch_predict_job.ModelBatchPredictOp(
+        project=project,
+        location=location,
+        model=get_vertex_model_task.outputs['artifact'],
+        job_display_name=(
+            f'autosxs-{name}-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}'
+        ),
+        gcs_source_uris=prediction_inputs,
+        instances_format='jsonl',
+        predictions_format='jsonl',
+        gcs_destination_output_uri_prefix=(
+            f'{dsl.PIPELINE_ROOT_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}'
+            f'/{name}_predictions'
+        ),
+        model_parameters=model_parameters,
+    )
+    prediction_uris_from_inference = function_based.get_uri(
+        artifact=batch_predict_task.outputs['gcs_output_directory'],
+        is_dir=True,
+    )
+
+  with dsl.Else(name='Responses Provided'):  # pylint: disable=singleton-comparison
+    prediction_uris_inference_provided = function_based.get_empty_string()
+
+  prediction_uris = dsl.OneOf(
+      prediction_uris_from_inference.output,
+      prediction_uris_inference_provided.output,
+  )
+
+  # We can't directly output dsl.OneOf, so we need to use identity.
+  return function_based.identity(x=prediction_uris).output
+
+
+# pylint: disable=dangerous-default-value,g-bare-generic
+@dsl.pipeline(
+    name='autosxs-template',
+    description='Determines the SxS winrate between two models.',
+)
+def autosxs_pipeline(
+    evaluation_dataset: str,
+    task: str,
+    id_columns: List[str],
+    model_a: str = '',
+    model_b: str = '',
+    autorater_prompt_parameters: Dict[str, Dict[str, str]] = {},  # pylint: disable=unused-argument
+    model_a_prompt_parameters: Dict[str, Dict[str, str]] = {},
+    model_b_prompt_parameters: Dict[str, Dict[str, str]] = {},
+    response_column_a: str = '',
+    response_column_b: str = '',
+    model_a_parameters: Dict[str, str] = {},
+    model_b_parameters: Dict[str, str] = {},
+    human_preference_column: str = '',
+    project: str = _placeholders.PROJECT_ID_PLACEHOLDER,
+    location: str = _placeholders.LOCATION_PLACEHOLDER,
+    judgments_format: str = 'jsonl',
+    bigquery_destination_prefix: str = '',
+    experimental_args: Dict[str, Any] = {},
+):
+  """Evaluates two models side-by-side using an arbiter model.
+
+  Args:
+    evaluation_dataset: A list of GCS paths to a JSONL dataset containing
+      evaluation examples.
+    task: Evaluation task in the form {task}@{version}. task can be one of
+      "summarization", "question_answer". Version is an integer with 3 digits or
+      "latest". Ex: summarization@001 or question_answer@latest.
+    id_columns: The columns which distinguish unique evaluation examples.
+    model_a: A fully-qualified model resource name. This parameter is optional
+      if Model A responses are specified.
+    model_b: A fully-qualified model resource name. This parameter is optional
+      if Model B responses are specified.
+    autorater_prompt_parameters: Map of autorater prompt parameters to columns
+      or templates. The expected parameters are: inference_instruction - Details
+      on how to perform a task. inference_context - Content to reference to
+      perform the task.
+    model_a_prompt_parameters: Map of Model A prompt template parameters to
+      columns or templates.
+    model_b_prompt_parameters: Map of Model B prompt template parameters to
+      columns or templates.
+    response_column_a: The column containing responses for model A. Required if
+      any response tables are provided for model A.
+    response_column_b: The column containing responses for model B. Required if
+      any response tables are provided for model B.
+    model_a_parameters: The parameters that govern the predictions from model A.
+    model_b_parameters: The parameters that govern the predictions from model B.
+    human_preference_column: The column containing ground truths. Only required
+      when users want to check the autorater alignment against human preference.
+    project: Project used to run custom jobs. Default is the same project used
+      to run the pipeline.
+    location: Location used to run custom jobs. Default is the same location
+      used to run the pipeline.
+    judgments_format: The format to write judgments to. Can be either 'json' or
+      'bigquery'.
+    bigquery_destination_prefix: BigQuery table to write judgments to if the
+      specified format is 'bigquery'.
+    experimental_args: Experimentally released arguments. Subject to change.
+  """
+  prediction_inputs_a = task_preprocess.task_preprocess(
+      evaluation_dataset=evaluation_dataset,
+      task=task,
+      model_prompt_parameters=model_a_prompt_parameters,
+      response_column=response_column_a,
+      human_preference_column=human_preference_column,
+      id_columns=id_columns,
+  ).set_display_name('Preprocess Model A Inputs')
+
+  prediction_inputs_b = task_preprocess.task_preprocess(
+      evaluation_dataset=evaluation_dataset,
+      task=task,
+      model_prompt_parameters=model_b_prompt_parameters,
+      response_column=response_column_b,
+      human_preference_column=human_preference_column,
+      id_columns=id_columns,
+  ).set_display_name('Preprocess Model B Inputs')
+
+  is_model_a_inference = function_based.get_usage_metric(
+      metadata=prediction_inputs_a.outputs['metadata'],
+      key='is_model_inference',
+  ).set_display_name('Read is_model_a_inference')
+
+  is_model_b_inference = function_based.get_usage_metric(
+      metadata=prediction_inputs_b.outputs['metadata'],
+      key='is_model_inference',
+  ).set_display_name('Read is_model_b_inference')
+
+  inferrer_a = _get_predictions(
+      name='A',
+      project=project,
+      location=location,
+      model=model_a,
+      model_parameters=model_a_parameters,
+      prediction_inputs=prediction_inputs_a.outputs['prediction_inputs'],
+      is_model_inference=is_model_a_inference.output,
+  ).set_display_name('Model A Responses')
+
+  inferrer_b = _get_predictions(
+      name='B',
+      project=project,
+      location=location,
+      model=model_b,
+      model_parameters=model_b_parameters,
+      prediction_inputs=prediction_inputs_b.outputs['prediction_inputs'],
+      is_model_inference=is_model_b_inference.output,
+  ).set_display_name('Model B Responses')
+
+  arbiter_input_preprocess = arbiter_preprocess.arbiter_preprocess(
+      autorater_prompt_parameters=autorater_prompt_parameters,
+      evaluation_dataset=evaluation_dataset,
+      id_columns=id_columns,
+      prediction_uris_b=inferrer_b.output,
+      prediction_uris_a=inferrer_a.output,
+      model_a_prompt_parameters=model_a_prompt_parameters,
+      model_b_prompt_parameters=model_b_prompt_parameters,
+      task=task,
+      response_column_a=response_column_a,
+      response_column_b=response_column_b,
+      human_preference_column=human_preference_column,
+      is_bp_output_a=is_model_a_inference.output,
+      is_bp_output_b=is_model_b_inference.output,
+  ).set_display_name('Preprocess Predictions')
+
+  autosxs_arbiter_task = autosxs_arbiter.autosxs_arbiter(
+      inference_output_uri=arbiter_input_preprocess.outputs[
+          'preprocessed_evaluation_dataset_uri'
+      ],
+      id_columns=id_columns,
+      human_preference_column=human_preference_column,
+      task=task,
+      judgments_format=judgments_format,
+      bigquery_destination_prefix=bigquery_destination_prefix,
+      experimental_args=experimental_args,
+  ).set_display_name('AutoSxS Arbiter')
+
+  has_human_preference = function_based.get_usage_metric(
+      metadata=prediction_inputs_a.outputs['metadata'],
+      key='has_human_preference_column',
+  ).set_display_name('Read has_human_preference_column')
+
+  autosxs_metrics_computer.autosxs_metrics_computer(
+      judgments_dir=autosxs_arbiter_task.outputs['judgments_uri'],
+      has_human_preference=has_human_preference.output,
+  ).set_display_name('AutoSxS Metrics')

google_cloud_pipeline_components/version.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Google Cloud Pipeline Components version."""
 
-__version__ = "2.6.0"
+__version__ = "2.7.0"

{google_cloud_pipeline_components-2.6.0.dist-info → google_cloud_pipeline_components-2.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-cloud-pipeline-components
-Version: 2.6.0
+Version: 2.7.0
 Summary: This SDK enables a set of First Party (Google owned) pipeline components that allow users to take their experience from Vertex AI SDK and other Google Cloud services and create a corresponding pipeline using KFP or Managed Pipelines.
 Home-page: https://github.com/kubeflow/pipelines/tree/master/components/google-cloud
 Author: The Google Cloud Pipeline Components authors