google-cloud-pipeline-components 2.6.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

google_cloud_pipeline_components/_implementation/model_evaluation/model_inference/component.py

@@ -0,0 +1,324 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Third party inference component."""
+from typing import Any, Dict, List, NamedTuple
+
+from google_cloud_pipeline_components import utils as gcpc_utils
+from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationTextGenerationOp
+from google_cloud_pipeline_components._implementation.model_evaluation import utils
+from kfp.dsl import Artifact
+from kfp.dsl import container_component
+from kfp.dsl import Metrics
+from kfp.dsl import Output
+from kfp.dsl import OutputPath
+from kfp.dsl import pipeline
+
+
+_IMAGE_URI = 'gcr.io/model-evaluation-dev/llm_eval:clyu-test'
+
+
+@container_component
+def model_inference_component_internal(
+    gcp_resources: OutputPath(str),
+    gcs_output_directory: Output[Artifact],
+    project: str,
+    location: str,
+    client_api_key_path: str,
+    prediction_instances_source_uri: str,
+    output_inference_gcs_prefix: str,
+    inference_platform: str = 'openai_chat_completions',
+    model_id: str = 'gpt-3.5-turbo',
+    request_params: Dict[str, Any] = {},
+    max_request_per_second: float = 3,
+    max_tokens_per_minute: float = 100,
+    display_name: str = 'third-party-inference',
+    machine_type: str = 'e2-highmem-16',
+    service_account: str = '',
+    network: str = '',
+    reserved_ip_ranges: List[str] = [],
+    encryption_spec_key_name: str = '',
+):
+  """Internal component to run Third Party Model Inference.
+
+  Args:
+      gcp_resources (str): Serialized gcp_resources proto tracking the custom
+        job.
+      model_inference_output_gcs_uri: The storage URI pointing toward a GCS
+        location to store CSV for third party inference.
+      project: Required. The GCP project that runs the pipeline component.
+      location: Required. The GCP region that runs the pipeline component.
+      client_api_key_path: The GCS URI where client API key.
+      output_inference_gcs_prefix: GCS file prefix for writing output.
+      display_name: display name of the pipeline.
+      machine_type: The machine type of this custom job. If not set, defaulted
+        to `e2-highmem-16`. More details:
+        https://cloud.google.com/compute/docs/machine-resource
+      service_account: Sets the default service account for workload run-as
+        account. The service account running the pipeline
+        (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account)
+        submitting jobs must have act-as permission on this run-as account. If
+        unspecified, the Vertex AI Custom Code Service
+        Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents)
+        for the CustomJob's project.
+      network: The full name of the Compute Engine network to which the job
+        should be peered. For example, projects/12345/global/networks/myVPC.
+        Format is of the form projects/{project}/global/networks/{network}.
+        Where {project} is a project number, as in 12345, and {network} is a
+        network name. Private services access must already be configured for the
+        network. If left unspecified, the job is not peered with any network.
+      reserved_ip_ranges: A list of names for the reserved ip ranges under the
+        VPC network that can be used for this job. If set, we will deploy the
+        job within the provided ip ranges. Otherwise, the job will be deployed
+        to any ip ranges under the provided VPC network.
+      encryption_spec_key_name: Customer-managed encryption key options for the
+        CustomJob. If this is set, then all resources created by the CustomJob
+        will be encrypted with the provided encryption key.
+
+  Returns:
+      gcp_resources (str): Serialized gcp_resources proto tracking the custom
+        job.
+      model_inference_output_gcs_uri: The storage URI pointing toward a
+        GCS location to store CSV for third party inference.
+  """
+  return gcpc_utils.build_serverless_customjob_container_spec(
+      project=project,
+      location=location,
+      custom_job_payload=utils.build_custom_job_payload(
+          display_name=display_name,
+          machine_type=machine_type,
+          image_uri=_IMAGE_URI,
+          args=[
+              f'--3p_model_inference={True}',
+              f'--project={project}',
+              f'--location={location}',
+              f'--prediction_instances_source_uri={prediction_instances_source_uri}',
+              f'--inference_platform={inference_platform}',
+              f'--output_inference_gcs_prefix={output_inference_gcs_prefix}',
+              f'--model_id={model_id}',
+              f'--request_params={request_params}',
+              f'--client_api_key_path={client_api_key_path}',
+              f'--max_request_per_second={max_request_per_second}',
+              f'--max_tokens_per_minute={max_tokens_per_minute}',
+              # f'--gcs_output_directory={gcs_output_directory}',
+              f'--gcs_output_directory={gcs_output_directory.path}',
+              '--executor_input={{$.json_escape[1]}}',
+          ],
+          service_account=service_account,
+          network=network,
+          reserved_ip_ranges=reserved_ip_ranges,
+          encryption_spec_key_name=encryption_spec_key_name,
+      ),
+      gcp_resources=gcp_resources,
+  )
+
+
+@pipeline(name='ModelEvaluationModelInferenceOp')
+def model_inference_component(
+    project: str,
+    location: str,
+    client_api_key_path: str,
+    prediction_instances_source_uri: str,
+    output_inference_gcs_prefix: str,
+    inference_platform: str = 'openai_chat_completions',
+    model_id: str = 'gpt-3.5-turbo',
+    request_params: Dict[str, Any] = {},
+    max_request_per_second: float = 3,
+    max_tokens_per_minute: float = 100,
+    display_name: str = 'third-party-inference',
+    machine_type: str = 'e2-highmem-16',
+    service_account: str = '',
+    network: str = '',
+    reserved_ip_ranges: List[str] = [],
+    encryption_spec_key_name: str = '',
+) -> NamedTuple(
+    'outputs',
+    gcs_output_directory=Artifact,
+):
+  """Component to run Third Party Model Inference.
+
+  Args:
+    project: Required. The GCP project that runs the pipeline component.
+      location: Required. The GCP region that runs the pipeline component.
+      client_api_key_path: The GCS URI where client API key.
+      output_inference_gcs_prefix: GCS file prefix for writing output.
+      display_name: display name of the pipeline.
+      machine_type: The machine type of this custom job. If not set, defaulted
+        to `e2-highmem-16`. More details:
+        https://cloud.google.com/compute/docs/machine-resource
+      service_account: Sets the default service account for workload run-as
+        account. The service account running the pipeline
+        (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account)
+        submitting jobs must have act-as permission on this run-as account. If
+        unspecified, the Vertex AI Custom Code Service
+        Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents)
+        for the CustomJob's project.
+      network: The full name of the Compute Engine network to which the job
+        should be peered. For example, projects/12345/global/networks/myVPC.
+        Format is of the form projects/{project}/global/networks/{network}.
+        Where {project} is a project number, as in 12345, and {network} is a
+        network name. Private services access must already be configured for the
+        network. If left unspecified, the job is not peered with any network.
+      reserved_ip_ranges: A list of names for the reserved ip ranges under the
+        VPC network that can be used for this job. If set, we will deploy the
+        job within the provided ip ranges. Otherwise, the job will be deployed
+        to any ip ranges under the provided VPC network.
+      encryption_spec_key_name: Customer-managed encryption key options for the
+        CustomJob. If this is set, then all resources created by the CustomJob
+        will be encrypted with the provided encryption key.
+
+  Returns:
+    NamedTuple:
+      model_inference_output_gcs_uri: CSV file output containing third
+      party prediction results.
+  """
+  outputs = NamedTuple(
+      'outputs',
+      gcs_output_directory=Artifact,
+  )
+
+  inference_task = model_inference_component_internal(
+      project=project,
+      location=location,
+      client_api_key_path=client_api_key_path,
+      prediction_instances_source_uri=prediction_instances_source_uri,
+      inference_platform=inference_platform,
+      model_id=model_id,
+      request_params=request_params,
+      max_request_per_second=max_request_per_second,
+      max_tokens_per_minute=max_tokens_per_minute,
+      output_inference_gcs_prefix=output_inference_gcs_prefix,
+      display_name=display_name,
+      machine_type=machine_type,
+      service_account=service_account,
+      network=network,
+      reserved_ip_ranges=reserved_ip_ranges,
+      encryption_spec_key_name=encryption_spec_key_name,
+  )
+
+  return outputs(
+      gcs_output_directory=inference_task.outputs['gcs_output_directory'],
+  )
+
+
+@pipeline(name='ModelEvaluationModelInferenceAndEvaluationPipeline')
+def model_inference_and_evaluation_component(
+    project: str,
+    location: str,
+    client_api_key_path: str,
+    prediction_instances_source_uri: str,
+    output_inference_gcs_prefix: str,
+    target_field_name: str = '',
+    inference_platform: str = 'openai_chat_completions',
+    model_id: str = 'gpt-3.5-turbo',
+    request_params: Dict[str, Any] = {},
+    max_request_per_second: float = 3,
+    max_tokens_per_minute: float = 100,
+    display_name: str = 'third-party-inference',
+    machine_type: str = 'e2-highmem-16',
+    service_account: str = '',
+    network: str = '',
+    reserved_ip_ranges: List[str] = [],
+    encryption_spec_key_name: str = '',
+) -> NamedTuple(
+    'outputs',
+    gcs_output_directory=Artifact,
+    evaluation_metrics=Metrics,
+):
+  """Component tun Third Party Model Inference and evaluation.
+
+  Args:
+    project: Required. The GCP project that runs the pipeline component.
+      location: Required. The GCP region that runs the pipeline component.
+      client_api_key_path: The GCS URI where client API key.
+      output_inference_gcs_prefix: GCS file prefix for writing output.
+      display_name: display name of the pipeline.
+      machine_type: The machine type of this custom job. If not set, defaulted
+        to `e2-highmem-16`. More details:
+        https://cloud.google.com/compute/docs/machine-resource
+      service_account: Sets the default service account for workload run-as
+        account. The service account running the pipeline
+        (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account)
+        submitting jobs must have act-as permission on this run-as account. If
+        unspecified, the Vertex AI Custom Code Service
+        Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents)
+        for the CustomJob's project.
+      network: The full name of the Compute Engine network to which the job
+        should be peered. For example, projects/12345/global/networks/myVPC.
+        Format is of the form projects/{project}/global/networks/{network}.
+        Where {project} is a project number, as in 12345, and {network} is a
+        network name. Private services access must already be configured for the
+        network. If left unspecified, the job is not peered with any network.
+      reserved_ip_ranges: A list of names for the reserved ip ranges under the
+        VPC network that can be used for this job. If set, we will deploy the
+        job within the provided ip ranges. Otherwise, the job will be deployed
+        to any ip ranges under the provided VPC network.
+      encryption_spec_key_name: Customer-managed encryption key options for the
+        CustomJob. If this is set, then all resources created by the CustomJob
+        will be encrypted with the provided encryption key.
+
+  Returns:
+    NamedTuple:
+      model_inference_output_gcs_uri: CSV file output containing third
+      party prediction results.
+  """
+  outputs = NamedTuple(
+      'outputs',
+      gcs_output_directory=Artifact,
+      evaluation_metrics=Metrics,
+  )
+
+  inference_task = model_inference_component_internal(
+      project=project,
+      location=location,
+      client_api_key_path=client_api_key_path,
+      prediction_instances_source_uri=prediction_instances_source_uri,
+      inference_platform=inference_platform,
+      model_id=model_id,
+      request_params=request_params,
+      max_request_per_second=max_request_per_second,
+      max_tokens_per_minute=max_tokens_per_minute,
+      output_inference_gcs_prefix=output_inference_gcs_prefix,
+      display_name=display_name,
+      machine_type=machine_type,
+      service_account=service_account,
+      network=network,
+      reserved_ip_ranges=reserved_ip_ranges,
+      encryption_spec_key_name=encryption_spec_key_name,
+  )
+
+  if inference_platform == 'openai_chat_completions':
+    prediction_field_name = 'predictions.0.message.content'
+  elif inference_platform == 'anthropic_predictions':
+    prediction_field_name = 'predictions'
+  else:
+    prediction_field_name = ''
+
+  eval_task = LLMEvaluationTextGenerationOp(
+      project=project,
+      location=location,
+      evaluation_task='text-generation',
+      target_field_name=target_field_name,
+      prediction_field_name=prediction_field_name,
+      predictions_format='jsonl',
+      joined_predictions_gcs_source=inference_task.outputs[
+          'gcs_output_directory'
+      ],
+      machine_type=machine_type,
+      encryption_spec_key_name=encryption_spec_key_name,
+  )
+
+  return outputs(
+      gcs_output_directory=inference_task.outputs['gcs_output_directory'],
+      evaluation_metrics=eval_task.outputs['evaluation_metrics'],
+  )

google_cloud_pipeline_components/_implementation/model_evaluation/version.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 """Version constants for model evaluation components."""
 
-_EVAL_VERSION = 'v0.9.3'
-_LLM_EVAL_VERSION = 'v0.3'
+_EVAL_VERSION = 'v0.9.4'
+_LLM_EVAL_VERSION = 'v0.5'
 
 _EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/model-evaluation'
 _LLM_EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/llm-model-evaluation'

google_cloud_pipeline_components/container/_implementation/model_evaluation/import_model_evaluation.py

@@ -56,6 +56,9 @@ parser = argparse.ArgumentParser(
     prog='Vertex Model Service evaluation importer', description=''
 )
 parser.add_argument('--metrics', dest='metrics', type=str, default='')
+parser.add_argument(
+    '--row_based_metrics', dest='row_based_metrics', type=str, default=''
+)
 parser.add_argument(
     '--classification_metrics',
     dest='classification_metrics',
@@ -274,6 +277,11 @@ def main(argv):
           'pipeline_job_resource_name': parsed_args.pipeline_job_resource_name,
           'evaluation_dataset_type': parsed_args.dataset_type,
           'evaluation_dataset_path': dataset_paths or None,
+          'row_based_metrics_path': (
+              parsed_args.row_based_metrics
+              if parsed_args.row_based_metrics
+              else None
+          ),
       }.items()
       if value
   }

google_cloud_pipeline_components/container/v1/automl_training_job/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Google Cloud Pipeline Components - AutoML Image Training Job container code."""

google_cloud_pipeline_components/container/v1/automl_training_job/image/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Google Cloud Pipeline Components - AutoML Image Training Job Launcher and Remote Runner."""

google_cloud_pipeline_components/container/v1/automl_training_job/image/launcher.py

@@ -0,0 +1,236 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""GCP launcher for AutoML image training jobs based on the AI Platform SDK."""
+
+import argparse
+import json
+import logging
+import sys
+from typing import List
+
+from google_cloud_pipeline_components.container.v1.automl_training_job.image import remote_runner
+from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import parser_util
+
+
+def _parse_args(args: List[str]):
+  """Parse command line arguments."""
+  args.append('--payload')
+  args.append('"{}"')  # Unused but required by parser_util.
+  parser, _ = parser_util.parse_default_args(args)
+  # Parse the conditionally required arguments
+  parser.add_argument(
+      '--display_name',
+      dest='display_name',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--prediction_type',
+      dest='prediction_type',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--multi_label',
+      dest='multi_label',
+      type=parser_util.parse_bool,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--model_type',
+      dest='model_type',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--labels',
+      dest='labels',
+      type=json.loads,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--dataset',
+      dest='dataset',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--disable_early_stopping',
+      dest='disable_early_stopping',
+      type=parser_util.parse_bool,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--training_encryption_spec_key_name',
+      dest='training_encryption_spec_key_name',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--model_encryption_spec_key_name',
+      dest='model_encryption_spec_key_name',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--model_display_name',
+      dest='model_display_name',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--training_fraction_split',
+      dest='training_fraction_split',
+      type=float,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--validation_fraction_split',
+      dest='validation_fraction_split',
+      type=float,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--test_fraction_split',
+      dest='test_fraction_split',
+      type=float,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--budget_milli_node_hours',
+      dest='budget_milli_node_hours',
+      type=int,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--training_filter_split',
+      dest='training_filter_split',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--validation_filter_split',
+      dest='validation_filter_split',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--test_filter_split',
+      dest='test_filter_split',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--base_model',
+      dest='base_model',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--incremental_train_base_model',
+      dest='incremental_train_base_model',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--parent_model',
+      dest='parent_model',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--is_default_version',
+      dest='is_default_version',
+      type=parser_util.parse_bool,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--model_version_aliases',
+      dest='model_version_aliases',
+      type=json.loads,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--model_version_description',
+      dest='model_version_description',
+      type=str,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parser.add_argument(
+      '--model_labels',
+      dest='model_labels',
+      type=json.loads,
+      required=False,
+      default=argparse.SUPPRESS,
+  )
+  parsed_args, _ = parser.parse_known_args(args)
+  args_dict = vars(parsed_args)
+  del args_dict['payload']
+  return args_dict
+
+
+def main(argv: List[str]):
+  """Main entry.
+
+  Expected input args are as follows:
+    Project - Required. The project of which the resource will be launched.
+    Region - Required. The region of which the resource will be launched.
+    Type - Required. GCP launcher is a single container. This Enum will
+        specify which resource to be launched.
+    gcp_resources - placeholder output for returning job_id.
+    Extra arguments - For constructing request payload. See remote_runner.py for
+    more information.
+
+  Args:
+    argv: A list of system arguments.
+  """
+  parsed_args = _parse_args(argv)
+  job_type = parsed_args['type']
+
+  if job_type != 'AutoMLImageTrainingJob':
+    raise ValueError('Incorrect job type: ' + job_type)
+
+  logging.info(
+      'Starting AutoMLImageTrainingJob using the following arguments: %s',
+      parsed_args,
+  )
+
+  remote_runner.create_pipeline(**parsed_args)
+
+
+if __name__ == '__main__':
+  main(sys.argv[1:])