datacommons-admin 0.0.1__py3-none-any.whl

datacommons_admin/__init__.py

@@ -0,0 +1 @@
+"""CLI package for administering Data Commons instances in GCP."""

datacommons_admin/admin_cli.py

@@ -0,0 +1,553 @@
+# Copyright 2026 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+import re
+import sys
+import urllib.request
+from typing import Any, Tuple
+
+import click
+from google.api_core import exceptions
+from google.cloud import storage
+
+from datacommons_admin.infra_templates import (
+    BACKEND_TF_TEMPLATE,
+    README_TEMPLATE,
+    REMOTE_STATE_TEMPLATE,
+)
+
+
+DEFAULT_BUCKET_LOCATION = "US"
+GITHUB_RAW_BASE_URL = "https://raw.githubusercontent.com/datacommonsorg/datacommons"
+GITHUB_REPO_URL = "https://github.com/datacommonsorg/datacommons.git"
+
+
+def _get_default_bucket_name(namespace: str, project_id: str) -> str:
+    """Returns the default Google Cloud Storage bucket name for Terraform state."""
+    return f"tf-state-{namespace}-{project_id}"
+
+
+def _get_default_state_prefix(namespace: str) -> str:
+    """Returns the default Google Cloud Storage object prefix for Terraform state."""
+    return f"terraform/state/{namespace}"
+
+
+def _log_resolved_value(label: str, value: str, is_default: bool, indent: int = 2):
+    """Logs a value with a bullet if default, or a green checkmark if from flag."""
+    prefix = " " * indent
+    padded_label = label.ljust(12)
+    if is_default:
+        click.echo(f"{prefix}- {padded_label}: {value} (Default)")
+    else:
+        click.secho(f"{prefix}✔", fg="green", nl=False)
+        click.echo(f" {padded_label}: {value} (from flag)")
+
+
+def _prompt(text: str, indent: int = 2, **kwargs):
+    """Prints the cyan [?] prompt symbol and calls click.prompt."""
+    click.secho(" " * indent + "[?]", fg="cyan", bold=True, nl=False)
+    prompt_text = text if text.startswith(" ") else f" {text}"
+    return click.prompt(prompt_text, **kwargs)
+
+
+def _confirm(text: str, indent: int = 2, **kwargs):
+    """Prints the cyan [?] prompt symbol and calls click.confirm."""
+    click.secho(" " * indent + "[?]", fg="cyan", bold=True, nl=False)
+    prompt_text = text if text.startswith(" ") else f" {text}"
+    return click.confirm(prompt_text, **kwargs)
+
+
+def _create_and_configure_bucket(
+    storage_client,
+    bucket_name: str,
+    project_id: str,
+    location: str = DEFAULT_BUCKET_LOCATION,
+) -> bool:
+    """Prompts and creates a Google Cloud Storage bucket, enables versioning, and sets IAM policy.
+
+    Returns True if created, False if cancelled.
+    """
+    click.echo(f"  - {'Status'.ljust(12)}: Not found")
+    click.echo(f"  - {'Project'.ljust(12)}: {project_id}")
+    _log_resolved_value("Location", location, location == DEFAULT_BUCKET_LOCATION)
+
+    if not _confirm("Create this bucket?", default=True):
+        return False
+
+    click.secho(
+        f"  Creating bucket gs://{bucket_name} in project {project_id} with location {location}...",
+        fg="bright_black",
+    )
+    new_bucket = storage_client.create_bucket(bucket_name, location=location)
+    new_bucket.iam_configuration.uniform_bucket_level_access_enabled = True
+    new_bucket.versioning_enabled = True
+    new_bucket.patch()
+    click.secho(f"  Enabling versioning...", fg="bright_black")
+    click.secho("  Configuring bucket IAM policy...", fg="bright_black")
+    policy = new_bucket.get_iam_policy(requested_policy_version=3)
+    policy["roles/storage.objectAdmin"].add(f"projectEditor:{project_id}")
+    policy["roles/storage.objectAdmin"].add(f"projectOwner:{project_id}")
+    new_bucket.set_iam_policy(policy)
+    return True
+
+
+def _abort_bucket_setup(is_default: bool):
+    click.secho(
+        "  No bucket available for remote Terraform state storage. Cancelling setup.",
+        fg="red",
+    )
+    if is_default:
+        click.secho(
+            "  Hint: Use --no-tf-remote-state to keep Terraform state local, or --tf-state-bucket to override the bucket name. See --help for more.",
+            fg="bright_black",
+        )
+    # Use sys.exit(1) instead of click.Abort() to avoid being caught by broad
+    # except Exception blocks in the caller and to avoid Click's "Aborted!" message.
+    sys.exit(1)
+
+
+def _ensure_bucket_ready(
+    storage_client,
+    bucket_name: str,
+    project_id: str,
+    location: str = DEFAULT_BUCKET_LOCATION,
+    is_default: bool = False,
+) -> bool:
+    """Checks if bucket exists and is ready to use, or creates it if missing.
+
+    Returns True if the bucket is ready to use, False if the user wants to try another name.
+    """
+    try:
+        bucket = storage_client.get_bucket(bucket_name)
+        click.echo(f"  - {'Status'.ljust(12)}: Found")
+        # Only prompt to reuse if it was the default bucket
+        if is_default:
+            if not _confirm("Use this bucket?", default=True):
+                _abort_bucket_setup(is_default)
+        else:
+            click.echo("  Proceeding...")
+        return True
+    except exceptions.NotFound:
+        if _create_and_configure_bucket(
+            storage_client, bucket_name, project_id, location
+        ):
+            return True
+        else:
+            _abort_bucket_setup(is_default)
+
+
+def _configure_remote_state(
+    project_id: str,
+    namespace: str,
+    bucket_name: str = "",
+    location: str = DEFAULT_BUCKET_LOCATION,
+) -> str:
+    """Handles Google Cloud Storage state bucket verification, creation, and IAM setup."""
+    try:
+        storage_client = storage.Client(project=project_id)
+    except Exception as e:
+        raise click.ClickException(
+            f"Failed to initialize Google Cloud Storage client for project '{project_id}': {e}. "
+            "Ensure you are authenticated via 'gcloud auth application-default login'."
+        )
+
+    is_default = False
+    if not bucket_name:
+        bucket_name = _get_default_bucket_name(namespace, project_id)
+        is_default = True
+
+    click.echo(
+        "Setting up Google Cloud Storage bucket for storing terraform state remotely:"
+    )
+    _log_resolved_value("Name", bucket_name, is_default)
+
+    try:
+        ready = _ensure_bucket_ready(
+            storage_client, bucket_name, project_id, location, is_default
+        )
+    except exceptions.Unauthorized as e:
+        raise click.ClickException(
+            f"Authentication failed: {e}\n"
+            "Please ensure you are authenticated. Run 'gcloud auth application-default login' and try again."
+        )
+    except exceptions.Forbidden as e:
+        raise click.ClickException(
+            f"Permission denied: {e}\n"
+            f"Please ensure your account has 'Storage Admin' or 'Project Editor' permissions in project '{project_id}'."
+        )
+    except click.Abort:
+        click.echo("")
+        sys.exit(1)
+    except Exception as e:
+        click.secho(
+            f"  Error: Failed to access or create bucket gs://{bucket_name}.",
+            fg="red",
+            bold=True,
+        )
+        click.secho(f"  {e}", fg="red")
+        raise click.ClickException("Setup cancelled.")
+
+    if not ready:
+        raise click.ClickException("Setup cancelled.")
+    return bucket_name
+
+
+def _get_github_templates(ref: str) -> tuple[str, str, str, str]:
+    """Fetches variables.tf, main.tf, outputs.tf, and terraform.tfvars.template from GitHub for the given ref."""
+    base_url = f"{GITHUB_RAW_BASE_URL}/{ref}/infra/dcp"
+
+    def fetch(filename: str) -> str:
+        url = f"{base_url}/{filename}"
+
+        req = urllib.request.Request(url, headers={"User-Agent": "DataCommons-CLI"})
+        with urllib.request.urlopen(req, timeout=10) as response:
+            return response.read().decode("utf-8")
+
+    return (
+        fetch("variables.tf"),
+        fetch("main.tf"),
+        fetch("outputs.tf"),
+        fetch("terraform.tfvars.template"),
+    )
+
+
+@click.group()
+def admin() -> None:
+    """Manage a Data Commons Platform instance in Google Cloud"""
+
+
+def _resolve_project_config(
+    project_id: str, namespace: str, force: bool
+) -> Tuple[str, str, Path]:
+    """Resolves project ID and namespace, and determines target directory."""
+    if project_id:
+        _log_resolved_value("Project ID", project_id, is_default=False)
+    if namespace:
+        _log_resolved_value("Namespace", namespace, is_default=False)
+
+    resolved_project_id = project_id.strip()
+    if not resolved_project_id:
+        resolved_project_id = _prompt(
+            "Google Cloud Platform project ID", type=str
+        ).strip()
+    if not resolved_project_id:
+        raise click.ClickException("GCP project ID must not be empty.")
+
+    resolved_namespace = namespace.strip()
+    while True:
+        if not resolved_namespace:
+            resolved_namespace = _prompt("Namespace", type=str).strip()
+            if not resolved_namespace:
+                click.secho("Error: Namespace must not be empty.", fg="red")
+                continue
+
+        target_dir = Path.cwd() / resolved_namespace
+        if target_dir.exists() and not force:
+            click.secho(
+                f"Error: Folder '{resolved_namespace}' already exists locally. "
+                "Please specify a different namespace, or use --force to overwrite.",
+                fg="yellow",
+            )
+            resolved_namespace = ""
+            continue
+
+        break
+
+    return resolved_project_id, resolved_namespace, target_dir
+
+
+def _check_existing_files(target_dir: Path, use_remote_state: bool, force: bool):
+    """Checks if target files already exist and raises error if they do (and not force)."""
+    main_tf_path = target_dir / "main.tf"
+    tfvars_path = target_dir / "terraform.tfvars"
+    readme_path = target_dir / "README.md"
+    backend_tf_path = target_dir / "backend.tf"
+
+    paths_to_check = [main_tf_path, tfvars_path, readme_path]
+    if use_remote_state:
+        paths_to_check.append(backend_tf_path)
+
+    existing_paths = [path for path in paths_to_check if path.exists()]
+    if existing_paths and not force:
+        existing_labels = ", ".join(str(path) for path in existing_paths)
+        raise click.ClickException(
+            f"Refusing to overwrite existing file(s): {existing_labels}. "
+            "Use --force to overwrite."
+        )
+
+
+def _setup_dcp_config_dir(
+    target_dir: Path,
+    project_id: str,
+    namespace: str,
+    bucket_name: str,
+    tf_state_prefix: str,
+    dc_api_key: str,
+    ref: str,
+    use_remote_state: bool,
+):
+    """Downloads and populates Terraform templates."""
+
+    api_key = dc_api_key.strip()
+    if not api_key:
+        api_key = _prompt(
+            "Data Commons API key (from apikeys.datacommons.org)",
+            type=str,
+            default="",
+            show_default=False,
+        ).strip()
+
+        if not api_key:
+            click.secho(
+                "  [!] Warning: Data Commons API key was skipped. You must add it to terraform.tfvars before running terraform apply.",
+                fg="yellow",
+                bold=True,
+            )
+
+    target_dir.mkdir(parents=True, exist_ok=True)
+    click.secho(f"Creating directory: {target_dir}", fg="bright_black")
+
+    try:
+        variables_content, main_content, outputs_content, tfvars_example = (
+            _get_github_templates(ref)
+        )
+
+        # Update the stack module source to point to GitHub
+        resolved_source = f"git::{GITHUB_REPO_URL}//infra/dcp/modules/stack?ref={ref}"
+        main_content = re.sub(
+            r'source\s*=\s*["\']\./modules/stack["\']',
+            f'source = "{resolved_source}"',
+            main_content,
+        )
+
+        # Write the files
+        (target_dir / "variables.tf").write_text(variables_content, encoding="utf-8")
+        (target_dir / "main.tf").write_text(main_content, encoding="utf-8")
+        (target_dir / "outputs.tf").write_text(outputs_content, encoding="utf-8")
+
+        # Modify tfvars_example with actual values
+        tfvars_content = tfvars_example
+        tfvars_content = tfvars_content.replace('"$$PROJECT_ID$$"', f'"{project_id}"')
+        tfvars_content = tfvars_content.replace('"$$NAMESPACE$$"', f'"{namespace}"')
+        if api_key:
+            tfvars_content = tfvars_content.replace('"$$DC_API_KEY$$"', f'"{api_key}"')
+
+        (target_dir / "terraform.tfvars").write_text(tfvars_content, encoding="utf-8")
+
+    except Exception as e:
+        raise click.ClickException(f"Failed to initialize Terraform templates: {e}")
+
+    remote_state_info = ""
+    if use_remote_state and bucket_name:
+        remote_state_info = REMOTE_STATE_TEMPLATE.format(
+            bucket_name=bucket_name, prefix=tf_state_prefix
+        )
+
+    (target_dir / "README.md").write_text(
+        README_TEMPLATE.format(remote_state_section=remote_state_info), encoding="utf-8"
+    )
+    if use_remote_state and bucket_name:
+        (target_dir / "backend.tf").write_text(
+            BACKEND_TF_TEMPLATE.format(
+                bucket_name=bucket_name,
+                prefix=tf_state_prefix,
+            ),
+            encoding="utf-8",
+        )
+
+    click.secho("Downloaded and populated Terraform templates.", fg="bright_black")
+
+
+@admin.command()
+@click.option(
+    "--project-id",
+    default="",
+    help="Google Cloud Platform project ID used for all resources related to your Data Commons instance.",
+)
+@click.option(
+    "--namespace", default="", help="Namespace prefix for provisioned resources."
+)
+@click.option("--dc-api-key", default="", help="Data Commons API key.")
+@click.option(
+    "--ref", default="main", show_default=True, help="Git ref for module source."
+)
+@click.option(
+    "--force", is_flag=True, help="Overwrite existing generated files if present."
+)
+@click.option(
+    "--tf-remote-state/--no-tf-remote-state",
+    default=True,
+    help="Enable or disable Terraform remote state management in Google Cloud Storage. Disabling ignores other --tf-state-* flags.",
+)
+@click.option(
+    "--tf-state-bucket",
+    default="",
+    help="Google Cloud Storage bucket for Terraform remote state. Generates a default name if omitted. Prompts to create the bucket if it is missing.",
+)
+@click.option(
+    "--tf-state-bucket-location",
+    default=DEFAULT_BUCKET_LOCATION,
+    show_default=True,
+    help="Google Cloud Storage bucket location if a new bucket needs to be created.",
+)
+@click.option(
+    "--tf-state-prefix",
+    default="",
+    help="Google Cloud Storage object prefix for Terraform state file (default: terraform/state/{namespace}).",
+)
+def init(
+    project_id: str,
+    namespace: str,
+    dc_api_key: str,
+    ref: str,
+    force: bool,
+    tf_remote_state: bool,
+    tf_state_bucket: str,
+    tf_state_bucket_location: str,
+    tf_state_prefix: str,
+) -> None:
+    """Initialize Terraform scaffolding for Data Commons administration/infrastructure."""
+    click.secho("Data Commons Admin Init", fg="cyan", bold=True)
+
+    # 1. Project Configs
+    click.secho("\n[Project Configuration]", fg="cyan", bold=True)
+    click.secho("Configuring project settings...", fg="bright_black")
+    resolved_project_id, resolved_namespace, target_dir = _resolve_project_config(
+        project_id, namespace, force
+    )
+
+    # 2. Terraform Setup
+    click.secho("\n[Terraform Backend Setup]", fg="cyan", bold=True)
+    click.secho("Configuring backend for Terraform state...", fg="bright_black")
+    if not tf_remote_state:
+        click.echo("  Using local backend for Terraform state.")
+
+    # Refuse to overwrite existing files unless --force is specified
+    _check_existing_files(target_dir, tf_remote_state, force)
+
+    resolved_bucket_name = (
+        _configure_remote_state(
+            resolved_project_id,
+            resolved_namespace,
+            tf_state_bucket,
+            tf_state_bucket_location,
+        )
+        if tf_remote_state
+        else ""
+    )
+
+    resolved_tf_state_prefix = tf_state_prefix.strip() or _get_default_state_prefix(
+        resolved_namespace
+    )
+
+    # 3. DCP config dir setup
+    click.secho("\n[DCP Configuration]", fg="cyan", bold=True)
+    click.secho("Setting up configuration files...", fg="bright_black")
+    _setup_dcp_config_dir(
+        target_dir,
+        resolved_project_id,
+        resolved_namespace,
+        resolved_bucket_name,
+        resolved_tf_state_prefix,
+        dc_api_key,
+        ref,
+        tf_remote_state,
+    )
+
+    click.secho(
+        f"Customize variables in {resolved_namespace}/terraform.tfvars as needed.",
+        fg="green",
+    )
+    click.secho(
+        f"Refer to {resolved_namespace}/README.md for more info and next steps.",
+        fg="green",
+    )
+
+
+def _setup_ingestion_client() -> Tuple[Any, str, str]:
+    click.secho(
+        "Fetching Ingestion Helper URI, Orchestrator Service Account, and Spanner details from Terraform outputs...",
+        fg="bright_black",
+    )
+
+    from datacommons_admin.tf_utils import (
+        get_dcp_ingestion_helper_uri,
+        get_dcp_orchestrator_service_account_email,
+        get_dcp_spanner_instance_id,
+        get_dcp_spanner_database_id,
+    )
+    from datacommons_admin.ingestion_helper_client import IngestionHelperClient
+
+    uri = get_dcp_ingestion_helper_uri()
+    sa_email = get_dcp_orchestrator_service_account_email()
+    instance_id = get_dcp_spanner_instance_id()
+    database_id = get_dcp_spanner_database_id()
+
+    click.secho(f"Found Ingestion Helper URI: {uri}", fg="green")
+    click.secho(f"Found Orchestrator Service Account: {sa_email}", fg="green")
+    click.secho(
+        f"Found Spanner Database Instance: {instance_id} / Database ID: {database_id}",
+        fg="green",
+    )
+
+    client = IngestionHelperClient(uri, service_account_email=sa_email)
+    return client, instance_id, database_id
+
+
+def _run_seed_db(client: Any, instance_id: str, database_id: str) -> None:
+    click.secho(
+        f"Seeding Spanner database '{instance_id}/{database_id}' via the Ingestion Helper service (this may take a few moments)...",
+        fg="bright_black",
+    )
+    result = client.seed_database()
+    click.secho("Successfully seeded Spanner database!", fg="green", bold=True)
+    if "message" in result:
+        click.secho(f"Details: {result['message']}", fg="bright_black")
+
+
+@admin.command(name="init-db")
+@click.option(
+    "--init-only", is_flag=True, help="Only initialize the database without seeding."
+)
+def init_db(init_only: bool) -> None:
+    """Initialize (and by default seed) the Spanner database via the DCP Ingestion Helper service."""
+    click.secho("Datacommons Admin Init-DB", fg="cyan", bold=True)
+    client, instance_id, database_id = _setup_ingestion_client()
+
+    click.secho(
+        f"Initializing Spanner database '{instance_id}/{database_id}' via the Ingestion Helper service (this may take a few moments)...",
+        fg="bright_black",
+    )
+    result = client.initialize_database()
+
+    click.secho("Successfully initialized Spanner database!", fg="green", bold=True)
+    if "message" in result:
+        click.secho(f"Details: {result['message']}", fg="bright_black")
+
+    if not init_only:
+        _run_seed_db(client, instance_id, database_id)
+
+
+@admin.command(name="seed-db")
+def seed_db() -> None:
+    """Seed the Spanner database via the DCP Ingestion Helper service."""
+    click.secho("Datacommons Admin Seed-DB", fg="cyan", bold=True)
+    client, instance_id, database_id = _setup_ingestion_client()
+    _run_seed_db(client, instance_id, database_id)
+
+
+from datacommons_admin.ingest_cli import ingest
+
+admin.add_command(ingest)

datacommons_admin/infra_templates.py

@@ -0,0 +1,75 @@
+# Copyright 2026 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+REMOTE_STATE_TEMPLATE = """
+## Remote State Management
+
+Terraform state is configured to be stored remotely in Google Cloud Storage:
+- **Bucket**: `gs://{bucket_name}`
+- **Prefix**: `{prefix}`
+
+This enables team collaboration, state locking, and automated pipeline deployments. The bucket is configured with Uniform Bucket-Level Access and object versioning.
+"""
+
+README_TEMPLATE = """# Data Commons Platform Terraform Setup
+
+This directory contains Terraform configuration for a new Data Commons Platform instance.
+
+Data Commons is an open knowledge graph for integrating and querying structured data across domains.
+The Data Commons Platform is the deployable infrastructure stack that runs Data Commons services in your GCP project.
+{remote_state_section}
+## What This Terraform Creates
+
+This setup deploys core Data Commons Platform infrastructure on GCP using the `infra/dcp` module, including Cloud Run services, Cloud Spanner resources, IAM bindings, and supporting service configuration.
+
+## Configure Variables
+
+Set environment-specific values in `terraform.tfvars` (for example `project_id`, `namespace`, and `dc_api_key`), and update module arguments in `main.tf` if you want to enable or tune additional features.
+
+## Learn More About Variables
+
+For the full list of supported module inputs and defaults, see:
+- https://github.com/datacommonsorg/datacommons/blob/main/infra/dcp/variables.tf
+- https://github.com/datacommonsorg/datacommons/blob/main/infra/dcp/outputs.tf
+- https://github.com/datacommonsorg/datacommons/blob/main/infra/dcp/terraform.tfvars.template
+- https://github.com/datacommonsorg/datacommons/blob/main/infra/dcp/README.md
+
+## Next Steps
+
+1. Review `terraform.tfvars` and update values if needed.
+2. Initialize Terraform:
+   ```bash
+   terraform init
+   ```
+3. Preview infrastructure changes:
+   ```bash
+   terraform plan
+   ```
+4. Deploy infrastructure:
+   ```bash
+   terraform apply
+   ```
+
+Generated using the Data Commons CLI tool:
+https://github.com/datacommonsorg/datacommons
+"""
+
+BACKEND_TF_TEMPLATE = """terraform {{
+  backend "gcs" {{
+    bucket = "{bucket_name}"
+    prefix = "{prefix}"
+  }}
+}}
+"""

datacommons_admin/ingest_cli.py

@@ -0,0 +1,142 @@
+# Copyright 2026 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import re
+
+from datacommons_admin.ingestion_job_client import IngestionJobClient
+from datacommons_admin.tf_utils import (
+    get_cdc_data_job_name,
+    get_dcp_orchestrator_service_account_email,
+    get_dcp_project_id,
+    get_dcp_region,
+    get_dcp_workflow_name,
+)
+
+
+@click.group(name="ingest")
+def ingest() -> None:
+    """Manage data ingestion jobs."""
+
+
+@ingest.command(name="start")
+def start() -> None:
+    """Start a data ingestion job execution."""
+    click.secho("Datacommons Admin Ingest Start", fg="cyan", bold=True)
+    click.secho(
+        "Fetching Data Job Name and Orchestrator Service Account from Terraform outputs...",
+        fg="bright_black",
+    )
+
+    job_name = get_cdc_data_job_name()
+    sa_email = get_dcp_orchestrator_service_account_email()
+    project_id = get_dcp_project_id()
+    region = get_dcp_region()
+    workflow_name = get_dcp_workflow_name()
+
+    click.secho(f"Found Data Job Name: {job_name}", fg="green")
+    click.secho(f"Found Orchestrator Service Account: {sa_email}", fg="green")
+    click.secho(f"Found GCP Project ID: {project_id}", fg="green")
+    click.secho(f"Found GCP Region: {region}", fg="green")
+    click.secho(
+        f"Starting Cloud Run job '{job_name}' via Admin API (this may take a few moments)...",
+        fg="bright_black",
+    )
+
+    client = IngestionJobClient(
+        job_name,
+        service_account_email=sa_email,
+        project_id=project_id,
+        location=region,
+    )
+    result = client.start_job()
+
+    click.secho("Successfully started ingestion job!", fg="green", bold=True)
+    res_name = result.get("name") or result.get("metadata", {}).get("name")
+
+    if res_name:
+        op_pattern = r"projects/([^/]+)/locations/([^/]+)/operations/([^/]+)"
+        op_match = re.match(op_pattern, res_name)
+
+        if op_match:
+            click.secho(f"Operation details: {res_name}", fg="bright_black")
+            resp_project_id, location, operation_id = op_match.groups()
+
+            short_job_name = job_name.split("/")[-1] if "/" in job_name else job_name
+            job_url = f"https://console.cloud.google.com/run/jobs/details/{location}/{short_job_name}/executions?project={resp_project_id}"
+
+            click.secho("Operation ID: ", fg="cyan", bold=True, nl=False)
+            click.secho(operation_id, fg="green")
+            click.secho("Job Console Link: ", fg="cyan", bold=True, nl=False)
+            click.secho(job_url, fg="blue", underline=True)
+        else:
+            click.secho(f"Resource details: {res_name}", fg="bright_black")
+
+        click.secho("\n[!] Note on Ingestion Completion", fg="yellow", bold=True)
+        click.secho(
+            "This job triggers a Cloud Workflow that runs in the background.\n"
+            "Check the Workflows console below to verify full completion.",
+            fg="yellow",
+        )
+
+        workflow_url = f"https://console.cloud.google.com/workflows/workflow/{region}/{workflow_name}/executions?project={project_id}"
+        click.secho("Workflow Console Link: ", fg="cyan", bold=True, nl=False)
+        click.secho(workflow_url, fg="blue", underline=True)
+
+
+@ingest.command(name="show-config")
+def show_config() -> None:
+    """Print the current ingestion job configuration (environment variables)."""
+    click.secho("Datacommons Admin Ingest Show-Config", fg="cyan", bold=True)
+    click.secho(
+        "Fetching Data Job Name and Orchestrator Service Account from Terraform outputs...",
+        fg="bright_black",
+    )
+
+    job_name = get_cdc_data_job_name()
+    sa_email = get_dcp_orchestrator_service_account_email()
+    project_id = get_dcp_project_id()
+    region = get_dcp_region()
+
+    click.secho(f"Found Data Job Name: {job_name}", fg="green")
+    click.secho(f"Found Orchestrator Service Account: {sa_email}", fg="green")
+    click.secho(f"Found GCP Project ID: {project_id}", fg="green")
+    click.secho(f"Found GCP Region: {region}", fg="green")
+    click.secho(
+        f"Fetching configuration for Cloud Run job '{job_name}'...",
+        fg="bright_black",
+    )
+
+    client = IngestionJobClient(
+        job_name,
+        service_account_email=sa_email,
+        project_id=project_id,
+        location=region,
+    )
+    env_vars = client.get_config()
+
+    click.secho("\nCurrent Ingestion Job Configuration:", fg="cyan", bold=True)
+    if not env_vars:
+        click.secho("No environment variables configured.", fg="yellow")
+    else:
+        for env in env_vars:
+            name = env.get("name", "UNKNOWN")
+            if "value" in env:
+                val = env["value"]
+            elif "valueSource" in env:
+                val = f"[SECRET: {env['valueSource']}]"
+            else:
+                val = "[UNSET]"
+            click.secho(f"  {name}: ", fg="bright_black", nl=False)
+            click.secho(str(val), fg="green")

datacommons_admin/ingestion_helper_client.py

@@ -0,0 +1,108 @@
+# Copyright 2026 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import google.auth
+from google.auth.transport.requests import AuthorizedSession, Request
+from google.oauth2 import id_token
+
+
+ACTION_INITIALIZE_DATABASE = "initialize_database"
+ACTION_SEED_DATABASE = "seed_database"
+
+
+class IngestionHelperClient:
+    """Client for interacting with the DCP Ingestion Helper Cloud Run service."""
+
+    def __init__(self, base_url: str, service_account_email: str = None) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.auth_req = Request()
+        self.service_account_email = service_account_email
+
+        base_credentials, _ = google.auth.default()
+
+        if service_account_email:
+            from google.auth import impersonated_credentials
+
+            target_credentials = impersonated_credentials.Credentials(
+                source_credentials=base_credentials,
+                target_principal=service_account_email,
+                target_scopes=["https://www.googleapis.com/auth/cloud-platform"],
+            )
+            creds = impersonated_credentials.IDTokenCredentials(
+                target_credentials=target_credentials,
+                target_audience=self.base_url,
+                include_email=True,
+            )
+        else:
+            try:
+                token = id_token.fetch_id_token(self.auth_req, self.base_url)
+                from google.oauth2.credentials import Credentials
+
+                creds = Credentials(token)
+            except Exception as e:
+                raise click.ClickException(
+                    f"Failed to fetch ID token for {self.base_url}: {e}\n"
+                    "Please ensure you are authenticated or provide a service account to impersonate."
+                )
+
+        self.session = AuthorizedSession(creds)
+
+    def _call_endpoint(self, action_type: str) -> dict:
+        url = self.base_url
+        payload = {"actionType": action_type}
+
+        try:
+            response = self.session.post(url, json=payload, timeout=300)
+        except Exception as e:
+            msg = f"Network or authentication error connecting to Ingestion Helper service at {url}: {e}"
+            if self.service_account_email:
+                msg += f"\nFailed to impersonate {self.service_account_email}. Please ensure your GCP user account has the 'Service Account Token Creator' (roles/iam.serviceAccountTokenCreator) IAM role."
+            raise click.ClickException(msg)
+
+        if response.status_code == 401:
+            raise click.ClickException(
+                f"HTTP 401 Unauthorized when calling Ingestion Helper at {url}.\n"
+                "Your GCP credentials were rejected. Please verify that the service account has the 'Cloud Run Invoker' (roles/run.invoker) IAM role for this service.\n"
+                "To re-authenticate, run:\n"
+                "  gcloud auth application-default login"
+            )
+
+        if not response.ok:
+            try:
+                error_data = response.json()
+                error_msg = (
+                    error_data.get("message")
+                    or error_data.get("error")
+                    or response.text
+                )
+            except Exception:
+                error_msg = response.text
+
+            raise click.ClickException(
+                f"Ingestion Helper returned HTTP {response.status_code}: {error_msg}"
+            )
+
+        try:
+            return response.json()
+        except Exception:
+            return {"status": "success", "message": response.text}
+
+    def initialize_database(self) -> dict:
+        """Calls the initialize_database endpoint on the ingestion helper service."""
+        return self._call_endpoint(ACTION_INITIALIZE_DATABASE)
+
+    def seed_database(self) -> dict:
+        """Calls the seed_database endpoint on the ingestion helper service."""
+        return self._call_endpoint(ACTION_SEED_DATABASE)

datacommons_admin/ingestion_job_client.py

@@ -0,0 +1,145 @@
+# Copyright 2026 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import google.auth
+from google.auth.transport.requests import AuthorizedSession
+
+
+class IngestionJobClient:
+    """Client for interacting with Cloud Run Admin API to manage CDC data ingestion jobs."""
+
+    def __init__(
+        self,
+        job_name: str,
+        service_account_email: str = None,
+        project_id: str = None,
+        location: str = None,
+    ) -> None:
+        self.service_account_email = service_account_email
+        base_credentials, _ = google.auth.default()
+
+        if not job_name.startswith("projects/"):
+            if not project_id:
+                raise click.ClickException(
+                    "Project ID must be provided via Terraform outputs or as an argument when job name is not a full resource name."
+                )
+            if not location:
+                raise click.ClickException(
+                    "Location must be provided via Terraform outputs or as an argument when job name is not a full resource name."
+                )
+            self.full_job_name = (
+                f"projects/{project_id}/locations/{location}/jobs/{job_name}"
+            )
+        else:
+            self.full_job_name = job_name
+
+        if service_account_email:
+            from google.auth import impersonated_credentials
+
+            creds = impersonated_credentials.Credentials(
+                source_credentials=base_credentials,
+                target_principal=service_account_email,
+                target_scopes=["https://www.googleapis.com/auth/cloud-platform"],
+            )
+        else:
+            creds = base_credentials
+
+        self.session = AuthorizedSession(creds)
+
+    def start_job(self) -> dict:
+        """Starts an execution of the Cloud Run job."""
+        url = f"https://run.googleapis.com/v2/{self.full_job_name}:run"
+        try:
+            response = self.session.post(url, json={}, timeout=300)
+        except Exception as e:
+            msg = f"Network or authentication error connecting to Cloud Run Admin API at {url}: {e}"
+            if self.service_account_email:
+                msg += f"\nFailed to impersonate {self.service_account_email}. Please ensure your GCP user account has the 'Service Account Token Creator' (roles/iam.serviceAccountTokenCreator) IAM role."
+            raise click.ClickException(msg)
+
+        if response.status_code == 401:
+            raise click.ClickException(
+                f"HTTP 401 Unauthorized when calling Cloud Run Admin API at {url}.\n"
+                "Your GCP credentials were rejected. Please verify your authentication.\n"
+                "To re-authenticate, run:\n"
+                "  gcloud auth application-default login"
+            )
+
+        if not response.ok:
+            try:
+                error_data = response.json()
+                error_msg = (
+                    error_data.get("message")
+                    or error_data.get("error", {}).get("message")
+                    or response.text
+                )
+            except Exception:
+                error_msg = response.text
+
+            raise click.ClickException(
+                f"Cloud Run Admin API returned HTTP {response.status_code}: {error_msg}"
+            )
+
+        try:
+            return response.json()
+        except Exception:
+            return {"status": "success", "message": response.text}
+
+    def get_config(self) -> list:
+        """Retrieves the environment variables configuration of the Cloud Run job."""
+        url = f"https://run.googleapis.com/v2/{self.full_job_name}"
+        try:
+            response = self.session.get(url, timeout=300)
+        except Exception as e:
+            msg = f"Network or authentication error connecting to Cloud Run Admin API at {url}: {e}"
+            if self.service_account_email:
+                msg += f"\nFailed to impersonate {self.service_account_email}. Please ensure your GCP user account has the 'Service Account Token Creator' (roles/iam.serviceAccountTokenCreator) IAM role."
+            raise click.ClickException(msg)
+
+        if response.status_code == 401:
+            raise click.ClickException(
+                f"HTTP 401 Unauthorized when calling Cloud Run Admin API at {url}.\n"
+                "Your GCP credentials were rejected. Please verify your authentication.\n"
+                "To re-authenticate, run:\n"
+                "  gcloud auth application-default login"
+            )
+
+        if not response.ok:
+            try:
+                error_data = response.json()
+                error_msg = (
+                    error_data.get("message")
+                    or error_data.get("error", {}).get("message")
+                    or response.text
+                )
+            except Exception:
+                error_msg = response.text
+
+            raise click.ClickException(
+                f"Cloud Run Admin API returned HTTP {response.status_code}: {error_msg}"
+            )
+
+        try:
+            job_data = response.json()
+        except Exception as e:
+            raise click.ClickException(f"Failed to parse Cloud Run job response: {e}")
+
+        containers = (
+            job_data.get("template", {}).get("template", {}).get("containers", [])
+        )
+        if not containers:
+            return []
+
+        return containers[0].get("env", [])

datacommons_admin/tf_utils.py

@@ -0,0 +1,134 @@
+# Copyright 2026 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import shutil
+import subprocess
+
+import click
+
+
+TF_OUTPUT_INGESTION_HELPER_URI = "dcp_ingestion_helper_uri"
+TF_OUTPUT_ORCHESTRATOR_SERVICE_ACCOUNT_EMAIL = "dcp_orchestrator_service_account_email"
+TF_OUTPUT_SPANNER_INSTANCE_ID = "dcp_spanner_instance_id"
+TF_OUTPUT_SPANNER_DATABASE_ID = "dcp_spanner_database_id"
+TF_OUTPUT_CDC_DATA_JOB_NAME = "cdc_data_job_name"
+TF_OUTPUT_PROJECT_ID = "project_id"
+TF_OUTPUT_REGION = "region"
+TF_OUTPUT_WORKFLOW_NAME = "workflow_name"
+
+
+def get_terraform_output(key: str) -> str:
+    """Fetches a specific key from `terraform output -json` with graceful error handling."""
+    if not shutil.which("terraform"):
+        raise click.ClickException(
+            "Terraform CLI not found. Please ensure Terraform is installed and available in your PATH."
+        )
+
+    try:
+        result = subprocess.run(
+            ["terraform", "output", "-json"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        raise click.ClickException(
+            f"Failed to run 'terraform output'. Are you in an initialized Terraform deployment directory?\n"
+            f"Error details: {e.stderr.strip() or e.stdout.strip()}"
+        )
+
+    try:
+        outputs = json.loads(result.stdout)
+    except json.JSONDecodeError:
+        raise click.ClickException(
+            "Failed to parse 'terraform output -json'. The output was not valid JSON."
+        )
+
+    if not outputs:
+        from pathlib import Path
+
+        cwd = Path.cwd()
+        has_tf_files = (
+            (cwd / ".terraform").exists()
+            or (cwd / "terraform.tfstate").exists()
+            or (cwd / "main.tf").exists()
+        )
+
+        if not has_tf_files:
+            raise click.ClickException(
+                f"No Terraform outputs found in '{cwd}'.\n"
+                "Please navigate to your initialized DCP Terraform directory (e.g., 'cd my-namespace') and ensure 'terraform apply' has been run."
+            )
+        else:
+            raise click.ClickException(
+                f"No Terraform outputs found in '{cwd}'.\n"
+                "Please ensure you have successfully run 'terraform apply' to generate the deployment state."
+            )
+
+    if key not in outputs:
+        from pathlib import Path
+
+        raise click.ClickException(
+            f"Terraform output key '{key}' not found in '{Path.cwd()}'.\n"
+            "Please verify that your Terraform configuration exports this output and that 'terraform apply' was fully completed."
+        )
+
+    value = outputs[key].get("value")
+    if not value:
+        raise click.ClickException(
+            f"Terraform output '{key}' is empty or null. Please verify your deployment state."
+        )
+
+    return str(value)
+
+
+def get_dcp_ingestion_helper_uri() -> str:
+    """Convenience wrapper to fetch the dcp_ingestion_helper_uri Terraform output."""
+    return get_terraform_output(TF_OUTPUT_INGESTION_HELPER_URI)
+
+
+def get_dcp_orchestrator_service_account_email() -> str:
+    """Convenience wrapper to fetch the dcp_orchestrator_service_account_email Terraform output."""
+    return get_terraform_output(TF_OUTPUT_ORCHESTRATOR_SERVICE_ACCOUNT_EMAIL)
+
+
+def get_dcp_spanner_instance_id() -> str:
+    """Convenience wrapper to fetch the dcp_spanner_instance_id Terraform output."""
+    return get_terraform_output(TF_OUTPUT_SPANNER_INSTANCE_ID)
+
+
+def get_dcp_spanner_database_id() -> str:
+    """Convenience wrapper to fetch the dcp_spanner_database_id Terraform output."""
+    return get_terraform_output(TF_OUTPUT_SPANNER_DATABASE_ID)
+
+
+def get_cdc_data_job_name() -> str:
+    """Convenience wrapper to fetch the cdc_data_job_name Terraform output."""
+    return get_terraform_output(TF_OUTPUT_CDC_DATA_JOB_NAME)
+
+
+def get_dcp_project_id() -> str:
+    """Convenience wrapper to fetch the project_id Terraform output."""
+    return get_terraform_output(TF_OUTPUT_PROJECT_ID)
+
+
+def get_dcp_region() -> str:
+    """Convenience wrapper to fetch the region Terraform output."""
+    return get_terraform_output(TF_OUTPUT_REGION)
+
+
+def get_dcp_workflow_name() -> str:
+    """Convenience wrapper to fetch the workflow_name Terraform output."""
+    return get_terraform_output(TF_OUTPUT_WORKFLOW_NAME)

datacommons_admin/version.py

@@ -0,0 +1,15 @@
+# Copyright 2026 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.0.1"

datacommons_admin-0.0.1.dist-info/METADATA

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: datacommons-admin
+Version: 0.0.1
+Summary: Data Commons Admin CLI
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/datacommonsorg/datacommons
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.1.7
+Requires-Dist: google-cloud-storage>=2.13.0
+Requires-Dist: pyopenssl>=24.0.0
+
+# datacommons-admin
+
+This package provides the `admin` command group consumed by the top-level `datacommons` CLI for initializing and managing Data Commons platform instances in GCP.

datacommons_admin-0.0.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+datacommons_admin/__init__.py,sha256=KFwbMH40mpcWY0d5sfAty6tGgGP6p_1ISu-_iKmCC70,67
+datacommons_admin/admin_cli.py,sha256=LxgiMuheQkUycsdkI24u7Qecg265_hn-HA0Eo68JzJk,19614
+datacommons_admin/infra_templates.py,sha256=ClStSytsSXgt-TkYfJU7y2bx1zCu5hsHL2QBawmzkUs,2717
+datacommons_admin/ingest_cli.py,sha256=KtV2GTeTq1ELmW4AS1WyEybaHWP8xaH9Y5FXGNxbxOQ,5399
+datacommons_admin/ingestion_helper_client.py,sha256=nGardf_n36GdQX0DPyIgu5rnaicAIIiAhYLFiMzBoNA,4415
+datacommons_admin/ingestion_job_client.py,sha256=gF1KvbphgzK7eiawM_kDxwvUJdxl0jQDCEwWCxvkEoM,5851
+datacommons_admin/tf_utils.py,sha256=vG8koG1kPZe8XjlSNGgGDMbPRFIcuhYhDljteUElaTk,4840
+datacommons_admin/version.py,sha256=oSC2MsGURWbdZseFaniLJiHDa6kjClfg_2shID77ZBA,598
+datacommons_admin-0.0.1.dist-info/METADATA,sha256=aHzjG8KinwUzYD8fM3sJT8M-ehyqCPEYF_EmO9I2yy4,546
+datacommons_admin-0.0.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+datacommons_admin-0.0.1.dist-info/top_level.txt,sha256=fRm2N5tIvdOXEPJ3aA7zLXOqNb9T4wOrewI2ayx2Qro,18
+datacommons_admin-0.0.1.dist-info/RECORD,,

datacommons_admin-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

datacommons_admin-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+datacommons_admin