google-cloud-pipeline-components 2.6.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

google_cloud_pipeline_components/_implementation/llm/task_preprocess.py

@@ -0,0 +1,97 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Component for preprocessing the evaluation dataset into prediction inputs."""
+
+import os
+from typing import Any, Dict, List
+
+from google_cloud_pipeline_components import _placeholders
+from google_cloud_pipeline_components import utils as gcpc_utils
+from google_cloud_pipeline_components._implementation.llm import utils
+from kfp import dsl
+
+
+def _resolve_image() -> str:
+  """Determines the image URI to create a container from."""
+  return (
+      os.environ.get('AUTOSXS_IMAGE_OVERRIDE')
+      or utils.get_default_image_uri('autosxs'))
+
+
+# pylint: disable=dangerous-default-value,g-bare-generic,unused-argument
+@dsl.container_component
+def task_preprocess(
+    evaluation_dataset: str,
+    id_columns: List[str],
+    task: str,
+    model_prompt_parameters: Dict[str, Dict[str, str]],
+    prediction_inputs: dsl.OutputPath(List[str]),  # pytype: disable=invalid-annotation
+    gcp_resources: dsl.OutputPath(str),  # pytype: disable=invalid-annotation
+    metadata: dsl.OutputPath(Dict[str, Any]),  # pytype: disable=invalid-annotation
+    response_column: str,
+    human_preference_column: str = '',
+) -> dsl.ContainerSpec:  # pylint: disable=g-doc-args
+  """Preprocesses evaluation dataset into prediction inputs.
+
+  Args:
+    evaluation_dataset: GCS or BigQuery URIs representing a dataset of prompts
+      and responses.
+    id_columns: The columns which distinguish unique evaluation examples.
+    task: Evaluation task in the form {task}@{version}. task can be one of
+      "summarization", "question_answer". Version is an integer with 3 digits or
+      "latest". Ex: summarization@001 or question_answer@latest.
+    model_prompt_parameters: Map of model prompt template parameters to columns
+      or templates.
+    response_column: Either an existing column containing predefined responses,
+      or the name of the model output column containing responses.
+    human_preference_column: The column containing ground truths. Only required
+      when users want to check the autorater alignment against human preference.
+
+  Returns:
+    prediction_inputs_path: Path to write the path where preprocessed
+      predictions are stored.
+    gcp_resources: Tracker for GCP resources created by this component.
+    metadata_path: Path to write the object that stores computed metrics
+      metadata for the task preprocess component.
+  """
+  return gcpc_utils.build_serverless_customjob_container_spec(
+      project=_placeholders.PROJECT_ID_PLACEHOLDER,
+      location=_placeholders.LOCATION_PLACEHOLDER,
+      custom_job_payload=utils.build_payload(
+          display_name='task_preprocess',
+          machine_type='n1-standard-4',
+          image_uri=_resolve_image(),
+          args=[
+              '--',  # Used to mark the start of component flags.
+              'task_preprocess',
+              f'--evaluation_dataset={evaluation_dataset}',
+              f'--staging_dir={dsl.PIPELINE_ROOT_PLACEHOLDER}',
+              f'--task={task}',
+              f'--prediction_inputs_path={prediction_inputs}',
+              (
+                  '--id_columns='
+                  "{{$.inputs.parameters['id_columns'].json_escape[0]}}"
+              ),
+              (
+                  '--model_prompt_parameters='
+                  "{{$.inputs.parameters['model_prompt_parameters']"
+                  '.json_escape[0]}}'
+              ),
+              f'--metadata_path={metadata}',
+              f'--response_column={response_column}',
+              f'--human_preference_column={human_preference_column}',
+          ],
+      ),
+      gcp_resources=gcp_resources,
+  )

google_cloud_pipeline_components/_implementation/llm/upload_llm_model.py

@@ -34,6 +34,7 @@ def upload_llm_model(
     gcp_resources: dsl.OutputPath(str),
     encryption_spec_key_name: str = '',
     upload_model: bool = True,
+    tune_type: str = '',
 ):
   """Uploads LLM model.
 
@@ -48,6 +49,8 @@ def upload_llm_model(
       upload_model: Whether to upload the model to the Model Registry. Default
         is ``True``. If ``False``, the model will not be uploaded and output
         artifacts will contain empty strings.
+      tune_type: Method used to tune the model, e.g. ``rlhf``. If present, this
+        value is used to set the ``tune-type`` run label during model upload.
 
   Returns:
       model_resource_name: Path to the created Model on Model Registry.
@@ -76,6 +79,8 @@ def upload_llm_model(
     labels['google-vertex-llm-tuning-base-model-id'] = (
         model_reference_name.replace('@', '-')
     )
+    if tune_type:
+      labels['tune-type'] = tune_type
 
     model_upload_payload = {
         'model': {

google_cloud_pipeline_components/_implementation/model_evaluation/__init__.py

@@ -35,6 +35,8 @@ from google_cloud_pipeline_components._implementation.model_evaluation.llm_infor
 from google_cloud_pipeline_components._implementation.model_evaluation.llm_retrieval_metrics.component import llm_retrieval_metrics as LLMRetrievalMetricsOp
 from google_cloud_pipeline_components._implementation.model_evaluation.llm_safety_bias.component import llm_safety_bias_metrics as LLMSafetyBiasMetricsOp
 from google_cloud_pipeline_components._implementation.model_evaluation.llm_safety_bias.evaluation_llm_safety_bias_pipeline import evaluation_llm_safety_bias_pipeline
+from google_cloud_pipeline_components._implementation.model_evaluation.model_inference.component import model_inference_and_evaluation_component
+from google_cloud_pipeline_components._implementation.model_evaluation.model_inference.component import model_inference_component
 from google_cloud_pipeline_components._implementation.model_evaluation.target_field_data_remover.component import target_field_data_remover as TargetFieldDataRemoverOp
 from google_cloud_pipeline_components._implementation.model_evaluation.text2sql.evaluation_llm_text2sql_pipeline import evaluation_llm_text2sql_pipeline
 
@@ -62,4 +64,6 @@ __all__ = [
     'ModelImportEvaluatedAnnotationOp',
     'ModelImportEvaluationOp',
     'TargetFieldDataRemoverOp',
+    'model_inference_component',
+    'model_inference_and_evaluation_component',
 ]

google_cloud_pipeline_components/_implementation/model_evaluation/endpoint_batch_predict/component.py

@@ -169,7 +169,7 @@ def evaluation_llm_endpoint_batch_predict_pipeline_graph_component(
     network: str = '',
     encryption_spec_key_name: str = '',
 ) -> NamedTuple('outputs', gcs_output_directory=Artifact):
-  """The LLM Evaluation Text2SQL Pipeline.
+  """The First Party Model Endpoint Batch Predict Pipeline.
 
   Args:
     project: Required. The GCP project that runs the pipeline components.

google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/component.py

@@ -31,6 +31,7 @@ def model_evaluation_import(
     gcp_resources: dsl.OutputPath(str),
     evaluation_resource_name: dsl.OutputPath(str),
     metrics: Optional[Input[Metrics]] = None,
+    row_based_metrics: Optional[Input[Metrics]] = None,
     problem_type: Optional[str] = None,
     classification_metrics: Optional[Input[ClassificationMetrics]] = None,
     forecasting_metrics: Optional[Input[ForecastingMetrics]] = None,
@@ -59,6 +60,8 @@ def model_evaluation_import(
     model: Vertex model resource that will be the parent resource of the
       uploaded evaluation.
     metrics: Path of metrics generated from an evaluation component.
+    row_based_metrics:
+      Path of row_based_metrics generated from an evaluation component.
     problem_type: The problem type of the metrics being imported to the
       VertexModel. `classification`, `regression`, `forecasting`,
       `text-generation`, `question-answering`, and `summarization` are the
@@ -106,6 +109,13 @@ def model_evaluation_import(
                   metrics.metadata["explanation_gcs_path"],
               ],
           ),
+          dsl.IfPresentPlaceholder(
+              input_name="row_based_metrics",
+              then=[
+                  "--row_based_metrics",
+                  row_based_metrics.uri,
+              ],
+          ),
           dsl.IfPresentPlaceholder(
               input_name="explanation",
               then=[

google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding/evaluation_llm_embedding_pipeline.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """LLM embedding evaluation pipeline based on information retrieval (IR) task."""
 
+from typing import Dict, Optional, Union
+from google_cloud_pipeline_components._implementation.model_evaluation.endpoint_batch_predict.component import evaluation_llm_endpoint_batch_predict_pipeline_graph_component as LLMEndpointBatchPredictOp
 from google_cloud_pipeline_components._implementation.model_evaluation.import_evaluation.component import model_evaluation_import as ModelImportEvaluationOp
 from google_cloud_pipeline_components._implementation.model_evaluation.llm_embedding_retrieval.component import llm_embedding_retrieval as LLMEmbeddingRetrievalOp
 from google_cloud_pipeline_components._implementation.model_evaluation.llm_information_retrieval_preprocessor.component import llm_information_retrieval_preprocessor as LLMInformationRetrievalPreprocessorOp
@@ -34,6 +36,8 @@ def evaluation_llm_embedding_pipeline(
     query_gcs_source: str,
     golden_docs_gcs_source: str,
     model_name: str,
+    qms_override: Optional[Dict[str, Union[int, float]]] = {},
+    model_parameters: Optional[Dict[str, Union[int, float]]] = {},
     embedding_chunking_function: str = 'langchain-RecursiveCharacterTextSplitter',
     embedding_chunk_size: int = 0,
     embedding_chunk_overlap: int = 0,
@@ -65,7 +69,14 @@ def evaluation_llm_embedding_pipeline(
     query_gcs_source: The gcs location for json file containing query documents.
     golden_docs_gcs_source: The gcs location for csv file containing mapping of
       each query to the golden docs.
-    model_name: The path for model to generate embeddings.
+    model_name: The path for model to generate embeddings, example,
+      'publishers/google/models/textembedding-gecko-multilingual@latest'
+    qms_override: Manual control of a large language model's qms. Write up when
+      there's an approved quota increase for a LLM. Write down when limiting qms
+      of a LLM for this pipeline. Should be provided as a dictionary, for
+      example {'text-bison': 20}. For deployed model which doesn't have
+      google-vertex-llm-tuning-base-model-id label, override the default here.
+    model_parameters: The parameters that govern the prediction.
     embedding_chunking_function: function used to split a document into chunks.
       Supported values are `langchain-RecursiveCharacterTextSplitter` and
       `sentence-splitter`. langchain-RecursiveCharacterTextSplitter:
@@ -157,38 +168,76 @@ def evaluation_llm_embedding_pipeline(
   )
   get_vertex_model_task.set_display_name('get-vertex-model')
 
-  batch_predict_corpus = ModelBatchPredictOp(
-      project=project,
+  batch_predict_corpus = LLMEndpointBatchPredictOp(
+      display_name=(
+          'batch-prediction-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}'
+      ),
+      publisher_model=model_name,
+      qms_override=qms_override,
       location=location,
-      model=get_vertex_model_task.outputs['artifact'],
-      job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}',
-      gcs_source_uris=preprocessing_task.outputs[
+      source_gcs_uris=preprocessing_task.outputs[
           'predictions_corpus_gcs_source'
       ],
-      instances_format=batch_predict_instances_format,
-      predictions_format=batch_predict_predictions_format,
+      model_parameters=model_parameters,
       gcs_destination_output_uri_prefix=(
           f'{PIPELINE_ROOT_PLACEHOLDER}/batch_predict_output'
       ),
+      service_account=service_account,
       encryption_spec_key_name=encryption_spec_key_name,
+      project=project,
   )
 
-  batch_predict_query = ModelBatchPredictOp(
-      project=project,
+  batch_predict_query = LLMEndpointBatchPredictOp(
+      display_name=(
+          'batch-prediction-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}'
+      ),
+      publisher_model=model_name,
+      qms_override=qms_override,
       location=location,
-      model=get_vertex_model_task.outputs['artifact'],
-      job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}',
-      gcs_source_uris=preprocessing_task.outputs[
+      source_gcs_uris=preprocessing_task.outputs[
           'predictions_query_gcs_source'
       ],
-      instances_format=batch_predict_instances_format,
-      predictions_format=batch_predict_predictions_format,
+      model_parameters=model_parameters,
       gcs_destination_output_uri_prefix=(
           f'{PIPELINE_ROOT_PLACEHOLDER}/batch_predict_output'
       ),
+      service_account=service_account,
       encryption_spec_key_name=encryption_spec_key_name,
+      project=project,
   )
 
+  # batch_predict_corpus = ModelBatchPredictOp(
+  #     project=project,
+  #     location=location,
+  #     model=get_vertex_model_task.outputs['artifact'],
+  #     job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}',
+  #     gcs_source_uris=preprocessing_task.outputs[
+  #         'predictions_corpus_gcs_source'
+  #     ],
+  #     instances_format=batch_predict_instances_format,
+  #     predictions_format=batch_predict_predictions_format,
+  #     gcs_destination_output_uri_prefix=(
+  #         f'{PIPELINE_ROOT_PLACEHOLDER}/batch_predict_output'
+  #     ),
+  #     encryption_spec_key_name=encryption_spec_key_name,
+  # )
+
+  # batch_predict_query = ModelBatchPredictOp(
+  #     project=project,
+  #     location=location,
+  #     model=get_vertex_model_task.outputs['artifact'],
+  #     job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}',
+  #     gcs_source_uris=preprocessing_task.outputs[
+  #         'predictions_query_gcs_source'
+  #     ],
+  #     instances_format=batch_predict_instances_format,
+  #     predictions_format=batch_predict_predictions_format,
+  #     gcs_destination_output_uri_prefix=(
+  #         f'{PIPELINE_ROOT_PLACEHOLDER}/batch_predict_output'
+  #     ),
+  #     encryption_spec_key_name=encryption_spec_key_name,
+  # )
+
   # TODO(b/290838262): Revisit if/when the concurrent jobs limit is increased/removed.
   batch_predict_query.after(batch_predict_corpus)
 

google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py

@@ -29,6 +29,7 @@ from kfp.dsl import OutputPath
 def model_evaluation_text_generation(
     gcp_resources: OutputPath(str),
     evaluation_metrics: Output[Metrics],
+    row_based_metrics: Output[Metrics],
     project: str,
     location: str,
     evaluation_task: str = 'text-generation',
@@ -38,6 +39,7 @@ def model_evaluation_text_generation(
     joined_predictions_gcs_source: dsl.Input[Artifact] = None,
     predictions_gcs_source: dsl.Input[Artifact] = None,
     ground_truth_gcs_source: str = '',
+    enable_row_based_metrics: bool = False,
     display_name: str = 'model-evaluation-text-generation',
     machine_type: str = 'e2-highmem-16',
     service_account: str = '',
@@ -106,11 +108,14 @@ def model_evaluation_text_generation(
       created.
 
   Returns:
-    evaluation_metrics: `Metrics` artifact representing the language model
-      evaluation metrics.
     gcp_resources: Serialized gcp_resources proto tracking the custom job.
       For more details, see
       https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.
+    evaluation_metrics: `Metrics` artifact representing the language model
+      evaluation metrics.
+    row_based_metrics: `Metrics` artifact representing the language model
+      evaluation metrics of each instance. This is only available if
+      enable_row_based_metrics is set to True.
   """
   return gcpc_utils.build_serverless_customjob_container_spec(
       project=project,
@@ -128,6 +133,8 @@ def model_evaluation_text_generation(
               f'--predictions_gcs_source={predictions_gcs_source.uri}',
               f'--ground_truth_gcs_source={ground_truth_gcs_source}',
               f'--evaluation_metrics_output_path={evaluation_metrics.path}',
+              f'--enable_row_based_metrics={enable_row_based_metrics}',
+              f'--row_based_metrics_output_path={row_based_metrics.path}',
               '--executor_input={{$.json_escape[1]}}',
           ],
           service_account=service_account,

google_cloud_pipeline_components/_implementation/model_evaluation/model_inference/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 The Kubeflow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Google Cloud Pipeline Evaluation Model Inference Component."""