agent-starter-pack 0.1.6__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of agent-starter-pack might be problematic. Click here for more details.
- {agent_starter_pack-0.1.6.dist-info → agent_starter_pack-0.2.0.dist-info}/METADATA +6 -6
- {agent_starter_pack-0.1.6.dist-info → agent_starter_pack-0.2.0.dist-info}/RECORD +78 -78
- agents/{agentic_rag_vertexai_search → agentic_rag}/README.md +3 -3
- agents/{agentic_rag_vertexai_search → agentic_rag}/app/agent.py +22 -6
- agents/agentic_rag/app/retrievers.py +132 -0
- agents/{agentic_rag_vertexai_search → agentic_rag}/notebooks/evaluating_langgraph_agent.ipynb +3 -3
- agents/{agentic_rag_vertexai_search → agentic_rag}/template/.templateconfig.yaml +3 -5
- agents/crewai_coding_crew/notebooks/evaluating_crewai_agent.ipynb +4 -4
- agents/crewai_coding_crew/notebooks/evaluating_langgraph_agent.ipynb +3 -3
- agents/langgraph_base_react/notebooks/evaluating_langgraph_agent.ipynb +3 -3
- agents/{multimodal_live_api → live_api}/README.md +7 -0
- agents/{multimodal_live_api → live_api}/app/agent.py +3 -11
- agents/{multimodal_live_api → live_api}/app/server.py +3 -2
- agents/{multimodal_live_api → live_api}/template/.templateconfig.yaml +2 -2
- src/base_template/Makefile +12 -7
- src/base_template/README.md +71 -71
- src/base_template/app/utils/tracing.py +3 -1
- src/base_template/app/utils/typing.py +1 -0
- src/base_template/deployment/cd/deploy-to-prod.yaml +10 -4
- src/base_template/deployment/cd/staging.yaml +11 -10
- src/base_template/deployment/ci/pr_checks.yaml +1 -1
- src/base_template/deployment/terraform/apis.tf +6 -0
- src/base_template/deployment/terraform/build_triggers.tf +34 -21
- src/base_template/deployment/terraform/dev/iam.tf +13 -6
- src/base_template/deployment/terraform/dev/log_sinks.tf +25 -28
- src/base_template/deployment/terraform/dev/providers.tf +1 -0
- src/base_template/deployment/terraform/dev/storage.tf +69 -11
- src/base_template/deployment/terraform/dev/variables.tf +50 -53
- src/base_template/deployment/terraform/dev/vars/env.tfvars +13 -11
- src/base_template/deployment/terraform/iam.tf +3 -3
- src/base_template/deployment/terraform/log_sinks.tf +24 -26
- src/base_template/deployment/terraform/providers.tf +2 -0
- src/base_template/deployment/terraform/service_accounts.tf +7 -7
- src/base_template/deployment/terraform/storage.tf +123 -11
- src/base_template/deployment/terraform/variables.tf +49 -70
- src/base_template/deployment/terraform/vars/env.tfvars +12 -17
- src/base_template/pyproject.toml +4 -3
- src/cli/commands/create.py +79 -19
- src/cli/commands/setup_cicd.py +91 -22
- src/cli/main.py +3 -1
- src/cli/utils/__init__.py +9 -2
- src/cli/utils/cicd.py +12 -0
- src/cli/utils/datastores.py +32 -0
- src/cli/utils/gcp.py +4 -6
- src/cli/utils/template.py +127 -45
- src/cli/utils/version.py +87 -0
- src/data_ingestion/README.md +24 -19
- src/data_ingestion/data_ingestion_pipeline/components/ingest_data.py +135 -2
- src/data_ingestion/data_ingestion_pipeline/components/process_data.py +276 -2
- src/data_ingestion/data_ingestion_pipeline/pipeline.py +28 -5
- src/data_ingestion/data_ingestion_pipeline/submit_pipeline.py +49 -14
- src/data_ingestion/pyproject.toml +1 -0
- src/deployment_targets/agent_engine/app/agent_engine_app.py +3 -1
- src/deployment_targets/cloud_run/tests/unit/test_server.py +15 -33
- src/frontends/live_api_react/frontend/package-lock.json +226 -186
- src/frontends/live_api_react/frontend/package.json +1 -1
- src/frontends/streamlit/frontend/utils/stream_handler.py +5 -5
- src/resources/containers/data_processing/Dockerfile +3 -1
- src/resources/locks/{uv-agentic_rag_vertexai_search-agent_engine.lock → uv-agentic_rag-agent_engine.lock} +747 -694
- src/resources/locks/{uv-agentic_rag_vertexai_search-cloud_run.lock → uv-agentic_rag-cloud_run.lock} +944 -806
- src/resources/locks/uv-crewai_coding_crew-agent_engine.lock +651 -694
- src/resources/locks/uv-crewai_coding_crew-cloud_run.lock +813 -789
- src/resources/locks/uv-langgraph_base_react-agent_engine.lock +666 -686
- src/resources/locks/uv-langgraph_base_react-cloud_run.lock +848 -798
- src/resources/locks/{uv-multimodal_live_api-cloud_run.lock → uv-live_api-cloud_run.lock} +856 -791
- src/resources/setup_cicd/cicd_variables.tf +5 -0
- src/resources/setup_cicd/github.tf +4 -2
- src/utils/watch_and_rebuild.py +14 -0
- agents/agentic_rag_vertexai_search/app/retrievers.py +0 -79
- src/deployment_targets/cloud_run/deployment/terraform/artifact_registry.tf +0 -22
- src/deployment_targets/cloud_run/deployment/terraform/dev/service_accounts.tf +0 -20
- {agent_starter_pack-0.1.6.dist-info → agent_starter_pack-0.2.0.dist-info}/WHEEL +0 -0
- {agent_starter_pack-0.1.6.dist-info → agent_starter_pack-0.2.0.dist-info}/entry_points.txt +0 -0
- {agent_starter_pack-0.1.6.dist-info → agent_starter_pack-0.2.0.dist-info}/licenses/LICENSE +0 -0
- /agents/{agentic_rag_vertexai_search → agentic_rag}/app/templates.py +0 -0
- /agents/{agentic_rag_vertexai_search → agentic_rag}/tests/integration/test_agent.py +0 -0
- /agents/{multimodal_live_api → live_api}/app/templates.py +0 -0
- /agents/{multimodal_live_api → live_api}/app/vector_store.py +0 -0
- /agents/{multimodal_live_api → live_api}/tests/integration/test_server_e2e.py +0 -0
- /agents/{multimodal_live_api → live_api}/tests/load_test/load_test.py +0 -0
- /agents/{multimodal_live_api → live_api}/tests/unit/test_server.py +0 -0
|
@@ -32,7 +32,14 @@ resource "google_project_iam_member" "default_compute_sa_storage_object_creator"
|
|
|
32
32
|
depends_on = [resource.google_project_service.services]
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
{
|
|
35
|
+
{% if cookiecutter.deployment_target == 'cloud_run' %}
|
|
36
|
+
resource "google_service_account" "cloud_run_app_sa" {
|
|
37
|
+
account_id = "${var.project_name}-cr"
|
|
38
|
+
display_name = "${var.project_name} Cloud Run App Service Account"
|
|
39
|
+
project = var.dev_project_id
|
|
40
|
+
depends_on = [resource.google_project_service.services]
|
|
41
|
+
}
|
|
42
|
+
|
|
36
43
|
# Grant Cloud Run SA the required permissions to run the application
|
|
37
44
|
resource "google_project_iam_member" "cloud_run_app_sa_roles" {
|
|
38
45
|
for_each = {
|
|
@@ -48,7 +55,7 @@ resource "google_project_iam_member" "cloud_run_app_sa_roles" {
|
|
|
48
55
|
member = "serviceAccount:${google_service_account.cloud_run_app_sa.email}"
|
|
49
56
|
depends_on = [resource.google_project_service.services]
|
|
50
57
|
}
|
|
51
|
-
{
|
|
58
|
+
{% elif cookiecutter.deployment_target == 'agent_engine' %}
|
|
52
59
|
# Grant required permissions to Vertex AI service account
|
|
53
60
|
resource "google_project_iam_member" "vertex_ai_sa_permissions" {
|
|
54
61
|
for_each = {
|
|
@@ -61,13 +68,13 @@ resource "google_project_iam_member" "vertex_ai_sa_permissions" {
|
|
|
61
68
|
member = google_project_service_identity.vertex_sa.member
|
|
62
69
|
depends_on = [resource.google_project_service.services]
|
|
63
70
|
}
|
|
64
|
-
{
|
|
65
|
-
{
|
|
71
|
+
{% endif %}
|
|
72
|
+
{% if cookiecutter.data_ingestion %}
|
|
66
73
|
# Service account to run Vertex AI pipeline
|
|
67
74
|
resource "google_service_account" "vertexai_pipeline_app_sa" {
|
|
68
75
|
for_each = local.project_ids
|
|
69
76
|
|
|
70
|
-
account_id = var.
|
|
77
|
+
account_id = "${var.project_name}-rag"
|
|
71
78
|
display_name = "Vertex AI Pipeline app SA"
|
|
72
79
|
project = each.value
|
|
73
80
|
depends_on = [resource.google_project_service.services]
|
|
@@ -87,4 +94,4 @@ resource "google_project_iam_member" "vertexai_pipeline_sa_roles" {
|
|
|
87
94
|
member = "serviceAccount:${google_service_account.vertexai_pipeline_app_sa[split(",", each.key)[0]].email}"
|
|
88
95
|
depends_on = [resource.google_project_service.services]
|
|
89
96
|
}
|
|
90
|
-
{
|
|
97
|
+
{% endif %}
|
|
@@ -19,26 +19,18 @@ resource "google_project_iam_member" "bigquery_data_editor" {
|
|
|
19
19
|
member = module.log_export_to_bigquery.writer_identity
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
log_sink_name = var.telemetry_sink_name
|
|
29
|
-
parent_resource_type = "project"
|
|
30
|
-
parent_resource_id = var.dev_project_id
|
|
31
|
-
destination_uri = "bigquery.googleapis.com/projects/${var.dev_project_id}/datasets/${var.telemetry_bigquery_dataset_id}"
|
|
32
|
-
filter = var.telemetry_logs_filter
|
|
33
|
-
bigquery_options = { use_partitioned_tables = true }
|
|
34
|
-
unique_writer_identity = true
|
|
35
|
-
depends_on = [resource.google_project_service.services]
|
|
22
|
+
resource "google_bigquery_dataset" "feedback_dataset" {
|
|
23
|
+
project = var.dev_project_id
|
|
24
|
+
dataset_id = replace("${var.project_name}_feedback", "-", "_")
|
|
25
|
+
friendly_name = "${var.project_name}_feedback"
|
|
26
|
+
location = var.region
|
|
27
|
+
depends_on = [resource.google_project_service.services]
|
|
36
28
|
}
|
|
37
29
|
|
|
38
|
-
resource "google_bigquery_dataset" "
|
|
30
|
+
resource "google_bigquery_dataset" "telemetry_logs_dataset" {
|
|
39
31
|
project = var.dev_project_id
|
|
40
|
-
dataset_id = var.
|
|
41
|
-
friendly_name = var.
|
|
32
|
+
dataset_id = replace("${var.project_name}_telemetry", "-", "_")
|
|
33
|
+
friendly_name = "${var.project_name}_telemetry"
|
|
42
34
|
location = var.region
|
|
43
35
|
depends_on = [resource.google_project_service.services]
|
|
44
36
|
}
|
|
@@ -46,21 +38,26 @@ resource "google_bigquery_dataset" "feedback_dataset" {
|
|
|
46
38
|
module "feedback_export_to_bigquery" {
|
|
47
39
|
source = "terraform-google-modules/log-export/google"
|
|
48
40
|
version = "10.0.0"
|
|
49
|
-
log_sink_name = var.
|
|
41
|
+
log_sink_name = "${var.project_name}_feedback"
|
|
50
42
|
parent_resource_type = "project"
|
|
51
43
|
parent_resource_id = var.dev_project_id
|
|
52
|
-
destination_uri = "bigquery.googleapis.com/projects/${var.dev_project_id}/datasets/${
|
|
44
|
+
destination_uri = "bigquery.googleapis.com/projects/${var.dev_project_id}/datasets/${google_bigquery_dataset.feedback_dataset.dataset_id}"
|
|
53
45
|
filter = var.feedback_logs_filter
|
|
54
46
|
bigquery_options = { use_partitioned_tables = true }
|
|
55
47
|
unique_writer_identity = true
|
|
56
|
-
depends_on = [
|
|
57
|
-
|
|
48
|
+
depends_on = [google_bigquery_dataset.feedback_dataset]
|
|
58
49
|
}
|
|
59
50
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
51
|
+
module "log_export_to_bigquery" {
|
|
52
|
+
source = "terraform-google-modules/log-export/google"
|
|
53
|
+
version = "10.0.0"
|
|
54
|
+
|
|
55
|
+
log_sink_name = "${var.project_name}_telemetry"
|
|
56
|
+
parent_resource_type = "project"
|
|
57
|
+
parent_resource_id = var.dev_project_id
|
|
58
|
+
destination_uri = "bigquery.googleapis.com/projects/${var.dev_project_id}/datasets/${google_bigquery_dataset.telemetry_logs_dataset.dataset_id}"
|
|
59
|
+
filter = var.telemetry_logs_filter
|
|
60
|
+
bigquery_options = { use_partitioned_tables = true }
|
|
61
|
+
unique_writer_identity = true
|
|
62
|
+
depends_on = [google_bigquery_dataset.telemetry_logs_dataset]
|
|
63
|
+
}
|
|
@@ -12,22 +12,23 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
provider "google" {
|
|
16
|
+
region = var.region
|
|
17
|
+
user_project_override = true
|
|
18
|
+
}
|
|
19
|
+
|
|
15
20
|
resource "google_storage_bucket" "logs_data_bucket" {
|
|
16
|
-
name = "${var.dev_project_id}-logs-data"
|
|
21
|
+
name = "${var.dev_project_id}-${var.project_name}-logs-data"
|
|
17
22
|
location = var.region
|
|
18
23
|
project = var.dev_project_id
|
|
19
24
|
uniform_bucket_level_access = true
|
|
20
25
|
|
|
21
|
-
lifecycle {
|
|
22
|
-
prevent_destroy = false
|
|
23
|
-
ignore_changes = all
|
|
24
|
-
}
|
|
25
26
|
depends_on = [resource.google_project_service.services]
|
|
26
27
|
}
|
|
27
28
|
|
|
28
|
-
{
|
|
29
|
+
{% if cookiecutter.data_ingestion %}
|
|
29
30
|
resource "google_storage_bucket" "data_ingestion_PIPELINE_GCS_ROOT" {
|
|
30
|
-
name = "${var.dev_project_id}-
|
|
31
|
+
name = "${var.dev_project_id}-${var.project_name}-rag"
|
|
31
32
|
location = var.region
|
|
32
33
|
project = var.dev_project_id
|
|
33
34
|
uniform_bucket_level_access = true
|
|
@@ -36,11 +37,12 @@ resource "google_storage_bucket" "data_ingestion_PIPELINE_GCS_ROOT" {
|
|
|
36
37
|
depends_on = [resource.google_project_service.services]
|
|
37
38
|
}
|
|
38
39
|
|
|
40
|
+
{% if cookiecutter.datastore_type == "vertex_ai_search" %}
|
|
39
41
|
resource "google_discovery_engine_data_store" "data_store_dev" {
|
|
40
42
|
location = var.data_store_region
|
|
41
43
|
project = var.dev_project_id
|
|
42
|
-
data_store_id = "${var.
|
|
43
|
-
display_name = "${var.
|
|
44
|
+
data_store_id = "${var.project_name}-datastore"
|
|
45
|
+
display_name = "${var.project_name}-datastore"
|
|
44
46
|
industry_vertical = "GENERIC"
|
|
45
47
|
content_config = "NO_CONTENT"
|
|
46
48
|
solution_types = ["SOLUTION_TYPE_SEARCH"]
|
|
@@ -51,7 +53,7 @@ resource "google_discovery_engine_data_store" "data_store_dev" {
|
|
|
51
53
|
|
|
52
54
|
resource "google_discovery_engine_search_engine" "search_engine_dev" {
|
|
53
55
|
project = var.dev_project_id
|
|
54
|
-
engine_id = "${var.
|
|
56
|
+
engine_id = "${var.project_name}-search"
|
|
55
57
|
collection_id = "default_collection"
|
|
56
58
|
location = google_discovery_engine_data_store.data_store_dev.location
|
|
57
59
|
display_name = "Search Engine App Staging"
|
|
@@ -61,4 +63,60 @@ resource "google_discovery_engine_search_engine" "search_engine_dev" {
|
|
|
61
63
|
}
|
|
62
64
|
provider = google.dev_billing_override
|
|
63
65
|
}
|
|
64
|
-
{
|
|
66
|
+
{% elif cookiecutter.datastore_type == "vertex_ai_vector_search" %}
|
|
67
|
+
resource "google_vertex_ai_index" "vector_search_index" {
|
|
68
|
+
project = var.dev_project_id
|
|
69
|
+
region = var.region
|
|
70
|
+
display_name = "${var.project_name}-vector-search"
|
|
71
|
+
description = "vector search index for test"
|
|
72
|
+
metadata {
|
|
73
|
+
config {
|
|
74
|
+
dimensions = var.vector_search_embedding_size
|
|
75
|
+
distance_measure_type = "DOT_PRODUCT_DISTANCE"
|
|
76
|
+
approximate_neighbors_count = var.vector_search_approximate_neighbors_count
|
|
77
|
+
shard_size = var.vector_search_shard_size
|
|
78
|
+
algorithm_config {
|
|
79
|
+
tree_ah_config {
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
index_update_method = "STREAM_UPDATE"
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
resource "google_vertex_ai_index_endpoint" "vector_search_index_endpoint" {
|
|
88
|
+
project = var.dev_project_id
|
|
89
|
+
region = var.region
|
|
90
|
+
display_name = "${var.project_name}-vector-search-endpoint"
|
|
91
|
+
public_endpoint_enabled = true
|
|
92
|
+
depends_on = [google_vertex_ai_index.vector_search_index]
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
resource "google_vertex_ai_index_endpoint_deployed_index" "vector_search_index_deployment" {
|
|
96
|
+
index_endpoint = google_vertex_ai_index_endpoint.vector_search_index_endpoint.id
|
|
97
|
+
index = google_vertex_ai_index.vector_search_index.id
|
|
98
|
+
deployed_index_id = replace("${var.project_name}_deployed_index", "-", "_")
|
|
99
|
+
dedicated_resources {
|
|
100
|
+
machine_spec {
|
|
101
|
+
machine_type = var.vector_search_machine_type
|
|
102
|
+
}
|
|
103
|
+
min_replica_count = var.vector_search_min_replica_count
|
|
104
|
+
max_replica_count = var.vector_search_max_replica_count
|
|
105
|
+
}
|
|
106
|
+
depends_on = [
|
|
107
|
+
google_vertex_ai_index.vector_search_index,
|
|
108
|
+
google_vertex_ai_index_endpoint.vector_search_index_endpoint
|
|
109
|
+
]
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
resource "google_storage_bucket" "vector_search_data_bucket" {
|
|
113
|
+
name = "${var.dev_project_id}-${var.project_name}-vs"
|
|
114
|
+
location = var.region
|
|
115
|
+
project = var.dev_project_id
|
|
116
|
+
uniform_bucket_level_access = true
|
|
117
|
+
force_destroy = true
|
|
118
|
+
|
|
119
|
+
depends_on = [resource.google_project_service.services]
|
|
120
|
+
}
|
|
121
|
+
{% endif %}
|
|
122
|
+
{% endif %}
|
|
@@ -12,6 +12,12 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
variable "project_name" {
|
|
16
|
+
type = string
|
|
17
|
+
description = "Project name used as a base for resource naming"
|
|
18
|
+
default = "{{ cookiecutter.project_name | replace('_', '-') }}"
|
|
19
|
+
}
|
|
20
|
+
|
|
15
21
|
variable "dev_project_id" {
|
|
16
22
|
type = string
|
|
17
23
|
description = "**Dev** Google Cloud Project ID for resource deployment."
|
|
@@ -23,54 +29,25 @@ variable "region" {
|
|
|
23
29
|
default = "us-central1"
|
|
24
30
|
}
|
|
25
31
|
|
|
26
|
-
variable "telemetry_bigquery_dataset_id" {
|
|
27
|
-
type = string
|
|
28
|
-
description = "BigQuery dataset ID for telemetry data export."
|
|
29
|
-
default = "telemetry_genai_app_sample_sink"
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
variable "feedback_bigquery_dataset_id" {
|
|
33
|
-
type = string
|
|
34
|
-
description = "BigQuery dataset ID for feedback data export."
|
|
35
|
-
default = "feedback_genai_app_sample_sink"
|
|
36
|
-
}
|
|
37
|
-
|
|
38
32
|
variable "telemetry_logs_filter" {
|
|
39
33
|
type = string
|
|
40
34
|
description = "Log Sink filter for capturing telemetry data. Captures logs with the `traceloop.association.properties.log_type` attribute set to `tracing`."
|
|
41
|
-
default = "jsonPayload.attributes.\"traceloop.association.properties.log_type\"=\"tracing\" jsonPayload.resource.attributes.\"service.name\"=\"
|
|
35
|
+
default = "jsonPayload.attributes.\"traceloop.association.properties.log_type\"=\"tracing\" jsonPayload.resource.attributes.\"service.name\"=\"{{cookiecutter.project_name}}\""
|
|
42
36
|
}
|
|
43
37
|
|
|
44
38
|
variable "feedback_logs_filter" {
|
|
45
39
|
type = string
|
|
46
40
|
description = "Log Sink filter for capturing feedback data. Captures logs where the `log_type` field is `feedback`."
|
|
47
|
-
default = "jsonPayload.log_type=\"feedback\""
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
variable "telemetry_sink_name" {
|
|
51
|
-
type = string
|
|
52
|
-
description = "Name of the telemetry data Log Sink."
|
|
53
|
-
default = "telemetry_logs_genai_app_sample"
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
variable "feedback_sink_name" {
|
|
57
|
-
type = string
|
|
58
|
-
description = "Name of the feedback data Log Sink."
|
|
59
|
-
default = "feedback_logs_genai_app_sample"
|
|
60
|
-
}
|
|
61
|
-
{%- if cookiecutter.deployment_target == 'cloud_run' %}
|
|
62
|
-
variable "cloud_run_app_sa_name" {
|
|
63
|
-
description = "Service account name to be used for the Cloud Run service"
|
|
64
|
-
type = string
|
|
65
|
-
default = "{{cookiecutter.project_name}}-cr"
|
|
41
|
+
default = "jsonPayload.log_type=\"feedback\" jsonPayload.service_name=\"{{cookiecutter.project_name}}\""
|
|
66
42
|
}
|
|
67
43
|
|
|
44
|
+
{% if cookiecutter.deployment_target == 'cloud_run' %}
|
|
68
45
|
variable "cloud_run_app_roles" {
|
|
69
46
|
description = "List of roles to assign to the Cloud Run app service account"
|
|
70
|
-
{
|
|
47
|
+
{% elif cookiecutter.deployment_target == 'agent_engine' %}
|
|
71
48
|
variable "agentengine_sa_roles" {
|
|
72
49
|
description = "List of roles to assign to the Agent Engine app service account"
|
|
73
|
-
{
|
|
50
|
+
{% endif %}
|
|
74
51
|
type = list(string)
|
|
75
52
|
default = [
|
|
76
53
|
"roles/aiplatform.user",
|
|
@@ -81,18 +58,7 @@ variable "agentengine_sa_roles" {
|
|
|
81
58
|
]
|
|
82
59
|
}
|
|
83
60
|
|
|
84
|
-
{
|
|
85
|
-
variable "vertexai_pipeline_sa_name" {
|
|
86
|
-
description = "Service account name to be used for the Vertex AI service"
|
|
87
|
-
type = string
|
|
88
|
-
default = "data-ingestion-vertexai-sa"
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
variable "data_store_region" {
|
|
92
|
-
type = string
|
|
93
|
-
description = "Google Cloud region for resource deployment."
|
|
94
|
-
default = "us"
|
|
95
|
-
}
|
|
61
|
+
{% if cookiecutter.data_ingestion %}
|
|
96
62
|
|
|
97
63
|
variable "pipelines_roles" {
|
|
98
64
|
description = "List of roles to assign to the Vertex AI runner service account"
|
|
@@ -111,16 +77,47 @@ variable "pipelines_roles" {
|
|
|
111
77
|
"roles/resourcemanager.projectIamAdmin"
|
|
112
78
|
]
|
|
113
79
|
}
|
|
80
|
+
{% if cookiecutter.datastore_type == "vertex_ai_search" %}
|
|
81
|
+
variable "data_store_region" {
|
|
82
|
+
type = string
|
|
83
|
+
description = "Google Cloud region for resource deployment."
|
|
84
|
+
default = "us"
|
|
85
|
+
}
|
|
86
|
+
{% elif cookiecutter.datastore_type == "vertex_ai_vector_search" %}
|
|
87
|
+
variable "vector_search_embedding_size" {
|
|
88
|
+
type = number
|
|
89
|
+
description = "The number of dimensions for the embeddings."
|
|
90
|
+
default = 768
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
variable "vector_search_approximate_neighbors_count" {
|
|
94
|
+
type = number
|
|
95
|
+
description = "The approximate number of neighbors to return."
|
|
96
|
+
default = 150
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
variable "vector_search_min_replica_count" {
|
|
100
|
+
type = number
|
|
101
|
+
description = "The min replica count for vector search instance"
|
|
102
|
+
default = 1
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
variable "vector_search_max_replica_count" {
|
|
106
|
+
type = number
|
|
107
|
+
description = "The max replica count for vector search instance"
|
|
108
|
+
default = 1
|
|
109
|
+
}
|
|
114
110
|
|
|
115
|
-
variable "
|
|
116
|
-
description = "The
|
|
111
|
+
variable "vector_search_shard_size" {
|
|
112
|
+
description = "The shard size of the vector search instance"
|
|
117
113
|
type = string
|
|
118
|
-
default = "
|
|
114
|
+
default = "SHARD_SIZE_SMALL"
|
|
119
115
|
}
|
|
120
116
|
|
|
121
|
-
variable "
|
|
122
|
-
description = "The
|
|
117
|
+
variable "vector_search_machine_type" {
|
|
118
|
+
description = "The machine type for the vector search instance"
|
|
123
119
|
type = string
|
|
124
|
-
default = "
|
|
120
|
+
default = "e2-standard-2"
|
|
125
121
|
}
|
|
126
|
-
{
|
|
122
|
+
{% endif %}
|
|
123
|
+
{% endif %}
|
|
@@ -1,21 +1,23 @@
|
|
|
1
|
+
# Project name used for resource naming
|
|
2
|
+
project_name = "{{ cookiecutter.project_name | replace('_', '-') }}"
|
|
3
|
+
|
|
1
4
|
# Your Dev Google Cloud project id
|
|
2
|
-
dev_project_id =
|
|
5
|
+
dev_project_id = "your-dev-project-id"
|
|
3
6
|
|
|
4
7
|
# The Google Cloud region you will use to deploy the infrastructure
|
|
5
8
|
region = "us-central1"
|
|
6
9
|
|
|
7
|
-
telemetry_bigquery_dataset_id = "telemetry_genai_app_sample_sink"
|
|
8
|
-
telemetry_sink_name = "telemetry_logs_genai_app_sample"
|
|
9
10
|
telemetry_logs_filter = "jsonPayload.attributes.\"traceloop.association.properties.log_type\"=\"tracing\" jsonPayload.resource.attributes.\"service.name\"=\"{{cookiecutter.project_name}}\""
|
|
10
|
-
|
|
11
|
-
feedback_bigquery_dataset_id = "feedback_genai_app_sample_sink"
|
|
12
|
-
feedback_sink_name = "feedback_logs_genai_app_sample"
|
|
13
11
|
feedback_logs_filter = "jsonPayload.log_type=\"feedback\""
|
|
14
|
-
{%- if cookiecutter.data_ingestion %}
|
|
15
|
-
search_engine_name = "sample-search-engine"
|
|
16
|
-
datastore_name = "sample-datastore"
|
|
17
|
-
vertexai_pipeline_sa_name = "vertexai-pipelines-sa"
|
|
18
12
|
|
|
19
|
-
|
|
13
|
+
{%- if cookiecutter.data_ingestion %}
|
|
14
|
+
{%- if cookiecutter.datastore_type == "vertex_ai_search" %}
|
|
15
|
+
# The value can only be one of "global", "us" and "eu".
|
|
20
16
|
data_store_region = "us"
|
|
17
|
+
{%- elif cookiecutter.datastore_type == "vertex_ai_vector_search" %}
|
|
18
|
+
vector_search_shard_size = "SHARD_SIZE_SMALL"
|
|
19
|
+
vector_search_machine_type = "e2-standard-2"
|
|
20
|
+
vector_search_min_replica_count = 1
|
|
21
|
+
vector_search_max_replica_count = 1
|
|
22
|
+
{%- endif %}
|
|
21
23
|
{%- endif %}
|
|
@@ -44,7 +44,7 @@ resource "google_project_iam_member" "other_projects_roles" {
|
|
|
44
44
|
member = "serviceAccount:${resource.google_service_account.cicd_runner_sa.email}"
|
|
45
45
|
depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
|
|
46
46
|
}
|
|
47
|
-
{
|
|
47
|
+
{% if cookiecutter.deployment_target == 'cloud_run' %}
|
|
48
48
|
# 3. Allow Cloud Run service SA to pull containers stored in the CICD project
|
|
49
49
|
resource "google_project_iam_member" "cicd_run_invoker_artifact_registry_reader" {
|
|
50
50
|
for_each = local.deploy_project_ids
|
|
@@ -71,7 +71,7 @@ resource "google_project_iam_member" "cloud_run_app_sa_roles" {
|
|
|
71
71
|
member = "serviceAccount:${google_service_account.cloud_run_app_sa[split(",", each.key)[0]].email}"
|
|
72
72
|
depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
|
|
73
73
|
}
|
|
74
|
-
{
|
|
74
|
+
{% elif cookiecutter.deployment_target == 'agent_engine' %}
|
|
75
75
|
resource "google_project_service_identity" "vertex_sa" {
|
|
76
76
|
for_each = local.deploy_project_ids
|
|
77
77
|
provider = google-beta
|
|
@@ -94,7 +94,7 @@ resource "google_project_iam_member" "vertex_ai_sa_permissions" {
|
|
|
94
94
|
member = "serviceAccount:service-${data.google_project.projects[split("_", each.key)[0]].number}@gcp-sa-aiplatform-re.iam.gserviceaccount.com"
|
|
95
95
|
depends_on = [resource.google_project_service.shared_services, resource.google_project_service_identity.vertex_sa]
|
|
96
96
|
}
|
|
97
|
-
{
|
|
97
|
+
{% endif %}
|
|
98
98
|
|
|
99
99
|
# Special assignment: Allow the CICD SA to create tokens
|
|
100
100
|
resource "google_service_account_iam_member" "cicd_run_invoker_token_creator" {
|
|
@@ -20,31 +20,38 @@ resource "google_project_iam_member" "bigquery_data_editor" {
|
|
|
20
20
|
member = module.log_export_to_bigquery[each.key].writer_identity
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
resource "google_bigquery_dataset" "feedback_dataset" {
|
|
24
|
+
for_each = local.deploy_project_ids
|
|
25
|
+
project = each.value
|
|
26
|
+
dataset_id = replace("${var.project_name}_feedback", "-", "_")
|
|
27
|
+
friendly_name = "${var.project_name}_feedback"
|
|
28
|
+
location = var.region
|
|
29
|
+
depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
resource "google_bigquery_dataset" "telemetry_logs_dataset" {
|
|
33
|
+
for_each = local.deploy_project_ids
|
|
34
|
+
project = each.value
|
|
35
|
+
dataset_id = replace("${var.project_name}_telemetry", "-", "_")
|
|
36
|
+
friendly_name = "${var.project_name}_telemetry"
|
|
37
|
+
location = var.region
|
|
38
|
+
depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
|
|
39
|
+
}
|
|
40
|
+
|
|
23
41
|
module "log_export_to_bigquery" {
|
|
24
42
|
for_each = local.deploy_project_ids
|
|
25
43
|
|
|
26
44
|
source = "terraform-google-modules/log-export/google"
|
|
27
45
|
version = "10.0.0"
|
|
28
46
|
|
|
29
|
-
log_sink_name = var.
|
|
47
|
+
log_sink_name = "${var.project_name}_telemetry"
|
|
30
48
|
parent_resource_type = "project"
|
|
31
49
|
parent_resource_id = each.value
|
|
32
|
-
destination_uri = "bigquery.googleapis.com/projects/${each.value}/datasets/${
|
|
50
|
+
destination_uri = "bigquery.googleapis.com/projects/${each.value}/datasets/${google_bigquery_dataset.telemetry_logs_dataset[each.key].dataset_id}"
|
|
33
51
|
filter = var.telemetry_logs_filter
|
|
34
52
|
bigquery_options = { use_partitioned_tables = true }
|
|
35
53
|
unique_writer_identity = true
|
|
36
|
-
depends_on = [
|
|
37
|
-
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
resource "google_bigquery_dataset" "feedback_dataset" {
|
|
41
|
-
for_each = local.deploy_project_ids
|
|
42
|
-
project = each.value
|
|
43
|
-
dataset_id = var.feedback_bigquery_dataset_id
|
|
44
|
-
friendly_name = var.feedback_bigquery_dataset_id
|
|
45
|
-
location = var.region
|
|
46
|
-
depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
|
|
47
|
-
|
|
54
|
+
depends_on = [google_bigquery_dataset.telemetry_logs_dataset]
|
|
48
55
|
}
|
|
49
56
|
|
|
50
57
|
module "feedback_export_to_bigquery" {
|
|
@@ -52,21 +59,12 @@ module "feedback_export_to_bigquery" {
|
|
|
52
59
|
|
|
53
60
|
source = "terraform-google-modules/log-export/google"
|
|
54
61
|
version = "10.0.0"
|
|
55
|
-
log_sink_name = var.
|
|
62
|
+
log_sink_name = "${var.project_name}_feedback"
|
|
56
63
|
parent_resource_type = "project"
|
|
57
64
|
parent_resource_id = each.value
|
|
58
|
-
destination_uri = "bigquery.googleapis.com/projects/${each.value}/datasets/${
|
|
65
|
+
destination_uri = "bigquery.googleapis.com/projects/${each.value}/datasets/${google_bigquery_dataset.feedback_dataset[each.key].dataset_id}"
|
|
59
66
|
filter = var.feedback_logs_filter
|
|
60
67
|
bigquery_options = { use_partitioned_tables = true }
|
|
61
68
|
unique_writer_identity = true
|
|
62
|
-
depends_on = [
|
|
69
|
+
depends_on = [google_bigquery_dataset.feedback_dataset]
|
|
63
70
|
}
|
|
64
|
-
|
|
65
|
-
resource "google_bigquery_dataset" "telemetry_logs_dataset" {
|
|
66
|
-
depends_on = [module.log_export_to_bigquery, module.feedback_export_to_bigquery, resource.google_project_service.shared_services]
|
|
67
|
-
for_each = local.deploy_project_ids
|
|
68
|
-
project = each.value
|
|
69
|
-
dataset_id = var.telemetry_bigquery_dataset_id
|
|
70
|
-
friendly_name = var.telemetry_bigquery_dataset_id
|
|
71
|
-
location = var.region
|
|
72
|
-
}
|
|
@@ -25,11 +25,13 @@ terraform {
|
|
|
25
25
|
provider "google" {
|
|
26
26
|
alias = "staging_billing_override"
|
|
27
27
|
billing_project = var.staging_project_id
|
|
28
|
+
region = var.region
|
|
28
29
|
user_project_override = true
|
|
29
30
|
}
|
|
30
31
|
|
|
31
32
|
provider "google" {
|
|
32
33
|
alias = "prod_billing_override"
|
|
33
34
|
billing_project = var.prod_project_id
|
|
35
|
+
region = var.region
|
|
34
36
|
user_project_override = true
|
|
35
37
|
}
|
|
@@ -13,30 +13,30 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
resource "google_service_account" "cicd_runner_sa" {
|
|
16
|
-
account_id = var.
|
|
16
|
+
account_id = "${var.project_name}-cb"
|
|
17
17
|
display_name = "CICD Runner SA"
|
|
18
18
|
project = var.cicd_runner_project_id
|
|
19
19
|
depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
|
|
20
20
|
}
|
|
21
|
-
{
|
|
21
|
+
{% if cookiecutter.deployment_target == 'cloud_run' %}
|
|
22
22
|
resource "google_service_account" "cloud_run_app_sa" {
|
|
23
23
|
for_each = local.deploy_project_ids
|
|
24
24
|
|
|
25
|
-
account_id = var.
|
|
25
|
+
account_id = "${var.project_name}-cr"
|
|
26
26
|
display_name = "Cloud Run Generative AI app SA"
|
|
27
27
|
project = each.value
|
|
28
28
|
depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
|
|
29
29
|
}
|
|
30
|
-
{
|
|
30
|
+
{% endif %}
|
|
31
31
|
|
|
32
|
-
{
|
|
32
|
+
{% if cookiecutter.data_ingestion %}
|
|
33
33
|
# Service account to run Vertex AI pipeline
|
|
34
34
|
resource "google_service_account" "vertexai_pipeline_app_sa" {
|
|
35
35
|
for_each = local.deploy_project_ids
|
|
36
36
|
|
|
37
|
-
account_id = var.
|
|
37
|
+
account_id = "${var.project_name}-rag"
|
|
38
38
|
display_name = "Vertex AI Pipeline app SA"
|
|
39
39
|
project = each.value
|
|
40
40
|
depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
|
|
41
41
|
}
|
|
42
|
-
{
|
|
42
|
+
{% endif %}
|