google-genai 1.53.0__py3-none-any.whl → 1.55.0__py3-none-any.whl

google/genai/tests/tunings/test_tune.py

@@ -0,0 +1,268 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+"""Tests for tunings.tune()."""
+
+from ... import types as genai_types
+from .. import pytest_helper
+import pytest
+
+evaluation_config=genai_types.EvaluationConfig(
+    metrics=[
+        genai_types.Metric(name="bleu", prompt_template="test prompt template")
+    ],
+    output_config=genai_types.OutputConfig(
+        gcs_destination=genai_types.GcsDestination(
+            output_uri_prefix="gs://sararob_test/"
+        )
+    ),
+)
+
+test_table: list[pytest_helper.TestTableItem] = [
+    pytest_helper.TestTableItem(
+        name="test_dataset_gcs_uri",
+        parameters=genai_types.CreateTuningJobParameters(
+            base_model="gemini-2.5-flash",
+            training_dataset=genai_types.TuningDataset(
+                gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_train_data.jsonl",
+            ),
+        ),
+        exception_if_mldev="gcs_uri parameter is not supported in Gemini API.",
+    ),
+    pytest_helper.TestTableItem(
+        name="test_tune_pretuned_model",
+        parameters=genai_types.CreateTuningJobParameters(
+            base_model="projects/801452371447/locations/us-central1/models/9030969596621881344",
+            training_dataset=genai_types.TuningDataset(
+                gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_train_data.jsonl",
+            ),
+        ),
+        exception_if_mldev="is not supported in Gemini API",
+    ),
+    pytest_helper.TestTableItem(
+        name="test_tune_pretuned_model_with_checkpoint_id",
+        parameters=genai_types.CreateTuningJobParameters(
+            base_model="projects/801452371447/locations/us-central1/models/9030969596621881344",
+            training_dataset=genai_types.TuningDataset(
+                gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_train_data.jsonl",
+            ),
+            config=genai_types.CreateTuningJobConfig(
+                pre_tuned_model_checkpoint_id="3",
+            ),
+        ),
+        exception_if_mldev="is not supported in Gemini API",
+    ),
+    pytest_helper.TestTableItem(
+        name="test_tune_simple_dpo",
+        parameters=genai_types.CreateTuningJobParameters(
+            base_model="gemini-2.5-flash",
+            training_dataset=genai_types.TuningDataset(
+                gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_train_data.jsonl",
+            ),
+            config=genai_types.CreateTuningJobConfig(
+                tuned_model_display_name="Model display name",
+                epoch_count=1,
+                method="PREFERENCE_TUNING",
+            ),
+        ),
+        exception_if_mldev="parameter is not supported in Gemini API.",
+    ),
+    pytest_helper.TestTableItem(
+        name="test_tune_dpo_with_beta",
+        parameters=genai_types.CreateTuningJobParameters(
+            base_model="gemini-2.5-flash",
+            training_dataset=genai_types.TuningDataset(
+                gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_train_data.jsonl",
+            ),
+            config=genai_types.CreateTuningJobConfig(
+                tuned_model_display_name="Model display name",
+                epoch_count=1,
+                method=genai_types.TuningMethod.PREFERENCE_TUNING,
+                beta=0.5,
+            ),
+        ),
+        exception_if_mldev="parameter is not supported in Gemini API.",
+    ),
+    pytest_helper.TestTableItem(
+        name="test_non_pretuned_model_with_checkpoint_id",
+        parameters=genai_types.CreateTuningJobParameters(
+            base_model="gemini-2.5-flash",
+            training_dataset=genai_types.TuningDataset(
+                gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_train_data.jsonl",
+            ),
+            config=genai_types.CreateTuningJobConfig(
+                pre_tuned_model_checkpoint_id="3",
+            ),
+        ),
+        exception_if_mldev="is not supported in Gemini API.",
+    ),
+    pytest_helper.TestTableItem(
+        name="test_dataset_gcs_uri_all_parameters",
+        parameters=genai_types.CreateTuningJobParameters(
+            base_model="gemini-2.5-flash",
+            training_dataset=genai_types.TuningDataset(
+                gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_train_data.jsonl",
+            ),
+            config=genai_types.CreateTuningJobConfig(
+                tuned_model_display_name="Model display name",
+                epoch_count=1,
+                learning_rate_multiplier=1.0,
+                adapter_size="ADAPTER_SIZE_ONE",
+                validation_dataset=genai_types.TuningValidationDataset(
+                    gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_validation_data.jsonl",
+                ),
+            ),
+        ),
+        exception_if_mldev="gcs_uri parameter is not supported in Gemini API.",
+    ),
+    pytest_helper.TestTableItem(
+        name="test_dataset_vertex_dataset_resource",
+        parameters=genai_types.CreateTuningJobParameters(
+            base_model="gemini-2.5-flash",
+            training_dataset=genai_types.TuningDataset(
+                vertex_dataset_resource="projects/801452371447/locations/us-central1/datasets/5779918772206829568",
+            ),
+        ),
+        exception_if_mldev="vertex_dataset_resource parameter is not supported in Gemini API.",
+    ),
+    pytest_helper.TestTableItem(
+        name="test_dataset_dataset_resource_all_parameters",
+        parameters=genai_types.CreateTuningJobParameters(
+            base_model="gemini-2.5-flash",
+            training_dataset=genai_types.TuningDataset(
+                vertex_dataset_resource="projects/801452371447/locations/us-central1/datasets/5779918772206829568",
+            ),
+            config=genai_types.CreateTuningJobConfig(
+                tuned_model_display_name="Model display name",
+                epoch_count=1,
+                learning_rate_multiplier=1.0,
+                adapter_size="ADAPTER_SIZE_ONE",
+                validation_dataset=genai_types.TuningValidationDataset(
+                    vertex_dataset_resource="projects/801452371447/locations/us-central1/datasets/1168232753779441664",
+                ),
+                labels={"testlabelkey": "testlabelvalue"},
+            ),
+        ),
+        exception_if_mldev="vertex_dataset_resource parameter is not supported in Gemini API.",
+    ),
+]
+
+pytestmark = pytest_helper.setup(
+    file=__file__,
+    globals_for_file=globals(),
+    test_method="tunings.tune",
+    test_table=test_table,
+)
+
+pytest_plugins = ("pytest_asyncio",)
+
+
+@pytest.mark.skipif(
+    "config.getoption('--private')",
+    reason="Skipping in pre-public tests"
+)
+def test_eval_config(client):
+  """Tests tuning with eval config."""
+
+  if client._api_client.vertexai:
+    tuning_job = client.tunings.tune(
+      base_model="gemini-2.5-flash",
+      training_dataset=genai_types.TuningDataset(gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_train_data.jsonl"),
+      config=genai_types.CreateTuningJobConfig(
+          tuned_model_display_name="tuning job with eval config",
+          epoch_count=1,
+          learning_rate_multiplier=1.0,
+          adapter_size="ADAPTER_SIZE_ONE",
+          validation_dataset=genai_types.TuningValidationDataset(
+              gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_validation_data.jsonl"
+          ),
+          evaluation_config=evaluation_config,
+      ),
+    )
+
+    assert tuning_job.state == genai_types.JobState.JOB_STATE_PENDING
+    assert tuning_job.evaluation_config == evaluation_config
+
+
+@pytest.mark.skipif(
+    "config.getoption('--private')",
+    reason="Skipping in pre-public tests"
+)
+def test_eval_config_with_metrics(client):
+  """Tests tuning with eval config metrics."""
+  if client._api_client.vertexai:
+    evaluation_config=genai_types.EvaluationConfig(
+        metrics=[
+            genai_types.Metric(
+                name="prompt-relevance",
+                prompt_template="How well does the response address the prompt?: PROMPT: {request}\n RESPONSE: {response}\n",
+                return_raw_output=True,
+                judge_model_system_instruction="You are a cat. Make all evaluations from this perspective.",
+            ),
+            genai_types.Metric(name='bleu'),
+            genai_types.Metric(name='rouge_1'),
+        ],
+        output_config=genai_types.OutputConfig(
+            gcs_destination=genai_types.GcsDestination(
+                output_uri_prefix="gs://sararob_test/"
+            )
+        ),
+        autorater_config=genai_types.AutoraterConfig(
+            sampling_count=1,
+            autorater_model="test-model",
+        ),
+    )
+    tuning_job = client.tunings.tune(
+      base_model="gemini-2.5-flash",
+      training_dataset=genai_types.TuningDataset(gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_train_data.jsonl"),
+      config=genai_types.CreateTuningJobConfig(
+          tuned_model_display_name="tuning job with eval config",
+          epoch_count=1,
+          learning_rate_multiplier=1.0,
+          adapter_size="ADAPTER_SIZE_ONE",
+          validation_dataset=genai_types.TuningValidationDataset(
+              gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_validation_data.jsonl"
+          ),
+          evaluation_config=evaluation_config,
+      ),
+    )
+    assert tuning_job.state == genai_types.JobState.JOB_STATE_PENDING
+
+
+@pytest.mark.skipif(
+    "config.getoption('--private')",
+    reason="Skipping in pre-public tests"
+)
+@pytest.mark.asyncio
+async def test_eval_config_async(client):
+  """Tests tuning with eval config."""
+  if client._api_client.vertexai:
+    tuning_job = await client.aio.tunings.tune(
+      base_model="gemini-2.5-flash",
+      training_dataset=genai_types.TuningDataset(gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_train_data.jsonl"),
+      config=genai_types.CreateTuningJobConfig(
+          tuned_model_display_name="tuning job with eval config",
+          epoch_count=1,
+          learning_rate_multiplier=1.0,
+          adapter_size="ADAPTER_SIZE_ONE",
+          validation_dataset=genai_types.TuningValidationDataset(
+              gcs_uri="gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_validation_data.jsonl"
+          ),
+          evaluation_config=evaluation_config,
+      ),
+    )
+
+    assert tuning_job.state == genai_types.JobState.JOB_STATE_PENDING

google/genai/tests/types/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+

google/genai/tests/types/test_bytes_internal.py

@@ -0,0 +1,271 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+import base64
+import json
+from unittest import mock
+
+import pytest
+
+from .. import pytest_helper
+
+from ... import _api_client as google_genai_api_client_module
+from ... import _common as common_module
+from ... import client as google_genai_client_module
+from ... import types
+
+
+# 64 chars in url safe base64.
+_BASE64_URL_SAFE = (
+    '-_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+)
+_RAW_BYTES = (
+    b'\xfb\xf6\x9bq\xd7\x9f\x82\x18\xa3\x92Y\xa7\xa2\x9a\xab\xb2\xdb\xaf\xc3\x1c\xb3\x00\x10\x83\x10Q\x87'
+    b' \x92\x8b0\xd3\x8fA\x14\x93QU\x97a\x9d5\xdb~9\xeb\xbf='
+)
+# 64 chars in normal base64 is invalid format for pydantic `val_json_bytes`.
+_BASE64_NOT_URL_SAFE = (
+    '+/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789==='
+)
+assert base64.b64decode(_BASE64_NOT_URL_SAFE) == _RAW_BYTES
+
+
+@pytest.fixture
+def client(use_vertex):
+  if use_vertex:
+    yield google_genai_client_module.Client(
+        vertexai=use_vertex, project='test-project', location='test-location'
+    )
+  else:
+    yield google_genai_client_module.Client(
+        vertexai=use_vertex, api_key='test-api-key'
+    )
+
+@pytest.fixture
+def encode_unserializable_types_method():
+  with mock.patch.object(common_module, 'encode_unserializable_types', wraps=common_module.encode_unserializable_types) as method:
+    yield method
+
+
+pytestmark = [pytest.mark.parametrize('use_vertex', [True, False])]
+
+
+@pytest.fixture
+def mock_request_method():
+  with mock.patch.object(
+      google_genai_api_client_module.BaseApiClient, 'request'
+  ) as request_method:
+    yield request_method
+
+
+# This test checks if user pass in valid base64 string(url safe base64)
+# via pydantic type, then SDK will return the raw bytes in pydantic type.
+@pytest.mark.usefixtures('client', 'mock_request_method', 'encode_unserializable_types_method')
+@pytest.mark.parametrize('bytes_input', [_RAW_BYTES, _BASE64_URL_SAFE])
+def test_base64_pydantic_input_success(
+    client, mock_request_method, encode_unserializable_types_method, bytes_input
+):
+  mock_request_method.return_value = types.HttpResponse(
+      headers={'header_key': 'header_value'},
+      body=json.dumps({
+          'candidates': [
+              {'content': {'parts': [{'text': 'Hello World'}], 'role': 'model'}}
+          ]
+      }),
+  )
+
+  response = client.models.generate_content(
+      model='gemini-2.5-flash-001',
+      contents=types.Content(
+          role='user',
+          parts=[
+              types.Part(
+                  inline_data=types.Blob(
+                      mime_type='image/png',
+                      data=bytes_input,
+                  )
+              )
+          ],
+      ),
+  )
+
+  encode_unserializable_types_method.assert_called()
+  assert mock_request_method.call_count == 1
+  assert (
+      pytest_helper.get_value_ignore_key_case(
+          mock_request_method.call_args[0][2]['contents'][0]['parts'][0],
+          'inlineData'
+      )['data']
+      == _BASE64_URL_SAFE
+  )
+  assert response.candidates[0].content == types.Content(
+      role='model',
+      parts=[types.Part(text='Hello World')],
+  )
+
+
+# This test checks if user pass in valid base64 string(url safe base64)
+# via dict type, then SDK will return the raw bytes in pydantic type.
+@pytest.mark.usefixtures('client', 'mock_request_method', 'encode_unserializable_types_method')
+@pytest.mark.parametrize('bytes_input', [_RAW_BYTES, _BASE64_URL_SAFE])
+def test_base64_dict_input_success(client, mock_request_method, encode_unserializable_types_method, bytes_input):
+  mock_request_method.return_value = types.HttpResponse(
+      headers={'header_key': 'header_value'},
+      body = json.dumps({
+          'candidates': [
+              {'content': {'parts': [{'text': 'Hello World'}], 'role': 'model'}}
+          ]
+      }),
+  )
+
+  response = client.models.generate_content(
+      model='gemini-2.5-flash-001',
+      contents={
+          'role': 'user',
+          'parts': [
+              {
+                  'inlineData': {
+                      'mimeType': 'image/png',
+                      'data': bytes_input,
+                  }
+              }
+          ],
+      },
+  )
+
+  encode_unserializable_types_method.assert_called()
+  assert mock_request_method.call_count == 1
+  assert (
+      pytest_helper.get_value_ignore_key_case(
+          mock_request_method.call_args[0][2]['contents'][0]['parts'][0],
+          'inlineData'
+      )['data']
+      == _BASE64_URL_SAFE
+  )
+  assert response.candidates[0].content == types.Content(
+      role='model',
+      parts=[types.Part(text='Hello World')],
+  )
+
+
+# This test checks if user pass in invalid base64 string(normal base64 but not
+# url safe) via pydantic type, then SDK will raise ValueError.
+@pytest.mark.usefixtures('client', 'mock_request_method')
+def test_base64_pydantic_input_failure(client):
+  with pytest.raises(ValueError, match='Data should be valid base64'):
+    client.models.generate_content(
+        model='gemini-2.5-flash-001',
+        contents=types.Content(
+            role='user',
+            parts=[
+                types.Part(
+                    inline_data=types.Blob(
+                        mime_type='image/png',
+                        data=_BASE64_NOT_URL_SAFE,
+                    )
+                )
+            ],
+        ),
+    )
+
+
+# This test checks if user pass in invalid base64 string(normal base64 but not
+# url safe) via dict type, then SDK will raise ValueError.
+@pytest.mark.usefixtures('client', 'mock_request_method')
+def test_base64_dict_input_failure(client):
+  with pytest.raises(ValueError, match='Data should be valid base64'):
+    client.models.generate_content(
+        model='gemini-2.5-flash-001',
+        contents={
+            'role': 'user',
+            'parts': [{
+                'inlineData': {
+                    'mimeType': 'image/png',
+                    'data': _BASE64_NOT_URL_SAFE,
+                }
+            }],
+        },
+    )
+
+
+# This test checks if server returns valid base64 string(url safe base64),
+# then SDK will return the raw bytes in pydantic type.
+@pytest.mark.usefixtures('client', 'mock_request_method',)
+def test_base64_pydantic_output_success(client, mock_request_method):
+  mock_request_method.return_value = types.HttpResponse(
+      headers={'header_key': 'header_value'},
+      body=json.dumps({
+          'candidates': [{
+              'content': {
+                  'parts': [{
+                      'inlineData': {
+                          'data': _BASE64_URL_SAFE,
+                          'mimeType': 'image/png',
+                      }
+                  }],
+                  'role': 'model',
+              }
+          }]
+      }),
+  )
+
+  response = client.models.generate_content(
+      model='gemini-2.5-flash-001',
+      contents=types.Content(
+          role='user',
+          parts=[types.Part(text='Hello World')],
+      ),
+  )
+
+  assert response.candidates[0].content == types.Content(
+      role='model',
+      parts=[
+          types.Part(
+              inline_data=types.Blob(data=_RAW_BYTES, mime_type='image/png')
+          )
+      ],
+  )
+
+
+# This test checks if server returns invalid base64 string(normal base64 but
+# not url safe), then SDK will raise ValueError.
+@pytest.mark.usefixtures('client', 'mock_request_method')
+def test_base64_pydantic_output_failure(client, mock_request_method):
+  mock_request_method.return_value = types.HttpResponse(
+      headers={'header_key': 'header_value'},
+      body=json.dumps({
+          'candidates': [{
+              'content': {
+                  'parts': [{
+                      'inlineData': {
+                          'data': _BASE64_NOT_URL_SAFE,
+                          'mimeType': 'image/png',
+                      }
+                  }],
+                  'role': 'model',
+              }
+          }]
+      }),
+  )
+
+  with pytest.raises(ValueError, match='Data should be valid base64'):
+    client.models.generate_content(
+        model='gemini-2.5-flash-001',
+        contents=types.Content(
+            role='user',
+            parts=[types.Part(text='Hello World')],
+        ),
+    )