dwe-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dwe/__init__.py +15 -0
- dwe/adapters.json +58 -0
- dwe/cli.py +591 -0
- dwe/git_ops.py +82 -0
- dwe/registry.py +143 -0
- dwe/secrets.py +180 -0
- dwe/service.py +294 -0
- dwe/state.py +54 -0
- dwe_core-0.1.0.dist-info/METADATA +738 -0
- dwe_core-0.1.0.dist-info/RECORD +14 -0
- dwe_core-0.1.0.dist-info/WHEEL +4 -0
- dwe_core-0.1.0.dist-info/entry_points.txt +3 -0
- dwe_core-0.1.0.dist-info/licenses/LICENSE +193 -0
- dwe_core-0.1.0.dist-info/licenses/NOTICE +4 -0
dwe/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright 2026 Ponder
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
|
dwe/adapters.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"dwe_cube": {
|
|
3
|
+
"url": "https://github.com/ponderedw/dwe_cube",
|
|
4
|
+
"type": "git",
|
|
5
|
+
"description": "Cube.js semantic layer + CubeStore + Milvus vector DB + RAG FastAPI",
|
|
6
|
+
"hub_name": "cube",
|
|
7
|
+
"display_name": "Cube.js Semantic Layer",
|
|
8
|
+
"icon": "boxes",
|
|
9
|
+
"git_providers": ["github", "gitlab"],
|
|
10
|
+
"cloud_providers": ["aws"],
|
|
11
|
+
"services": [
|
|
12
|
+
{"name": "CubeJS", "description": "Semantic layer \u2014 REST / GraphQL / WebSocket API over your data warehouse"},
|
|
13
|
+
{"name": "CubeStore", "description": "Pre-aggregation cache for sub-second query responses"},
|
|
14
|
+
{"name": "Milvus", "description": "Vector database for semantic / similarity search"},
|
|
15
|
+
{"name": "RAG FastAPI", "description": "Retrieval-augmented generation API backed by Milvus"},
|
|
16
|
+
{"name": "dbt-cube-sync", "description": "Syncs dbt models to Cube.js schema definitions automatically"},
|
|
17
|
+
{"name": "Application Load Balancer", "description": "HTTP -> HTTPS redirect + HTTPS termination. Configurable as internet-facing or internal (VPC-only) via ALB_INTERNAL secret"},
|
|
18
|
+
{"name": "DNS", "description": "Route53 CNAME record pointing to the ALB"},
|
|
19
|
+
{"name": "EC2 DNS", "description": "Route53 A record pointing directly to the EC2 instance IP — private IP for internal ALB, public IP for internet-facing"}
|
|
20
|
+
],
|
|
21
|
+
"required_secrets": [
|
|
22
|
+
{"key": "AWS_ACCESS_KEY_ID", "description": "AWS access key \u2014 Pulumi uses this to provision infrastructure and SSM to redeploy the app", "destination": "ci"},
|
|
23
|
+
{"key": "AWS_SECRET_ACCESS_KEY", "description": "AWS secret key", "destination": "ci"},
|
|
24
|
+
{"key": "PULUMI_S3_STATE", "description": "S3 URL for Pulumi state backend, e.g. s3://my-pulumi-state-bucket", "destination": "ci"},
|
|
25
|
+
{"key": "ROUTE53_ZONE_ID", "description": "Route53 hosted zone ID (e.g. Z1D633PJN98FT9)", "destination": "secrets_manager"},
|
|
26
|
+
{"key": "DNS_NAME", "description": "Public DNS name for the Cube API (e.g. cube.myorg.com)", "destination": "secrets_manager"},
|
|
27
|
+
{"key": "ACM_CERTIFICATE_ARN", "description": "ACM certificate ARN for HTTPS on the ALB (must cover DNS_NAME)", "destination": "secrets_manager"},
|
|
28
|
+
{"key": "VPC_ID", "description": "AWS VPC ID where infrastructure is deployed", "destination": "secrets_manager"},
|
|
29
|
+
{"key": "ALB_SUBNET_IDS", "description": "JSON array of public subnet IDs for the ALB (e.g. [\"subnet-aaa\",\"subnet-bbb\"])", "destination": "secrets_manager"},
|
|
30
|
+
{"key": "KEY_NAME", "description": "EC2 key pair name for SSH access", "destination": "secrets_manager"},
|
|
31
|
+
{"key": "EC2_SECURITY_GROUP_ID", "description": "Additional EC2 security group ID to attach (e.g. existing SG with DB/SSM rules)", "destination": "secrets_manager"},
|
|
32
|
+
{"key": "LB_SECURITY_GROUP_ID", "description": "Security group ID to attach to the load balancer (e.g. internal SG restricting access)", "destination": "secrets_manager"},
|
|
33
|
+
{"key": "ALB_INTERNAL", "description": "Set to 'true' for internal ALB (VPC-only), 'false' for internet-facing. Default: false", "destination": "secrets_manager"},
|
|
34
|
+
{"key": "git_deploy_token", "description": "GitLab / GitHub token — EC2 uses this to clone the repo on first boot", "destination": "secrets_manager"},
|
|
35
|
+
{"key": "git_deploy_username", "description": "Username paired with git_deploy_token for HTTPS clone", "destination": "secrets_manager"},
|
|
36
|
+
{"key": "CUBEJS_DB_TYPE", "description": "Data warehouse driver (postgres | bigquery | snowflake | athena | redshift | trino \u2026)", "destination": "secrets_manager"},
|
|
37
|
+
{"key": "CUBEJS_DB_HOST", "description": "Data warehouse hostname", "destination": "secrets_manager"},
|
|
38
|
+
{"key": "CUBEJS_DB_PORT", "description": "Data warehouse port (e.g. 5432 for Postgres)", "destination": "secrets_manager"},
|
|
39
|
+
{"key": "CUBEJS_DB_NAME", "description": "Data warehouse database / catalog name", "destination": "secrets_manager"},
|
|
40
|
+
{"key": "CUBEJS_DB_USER", "description": "Data warehouse username", "destination": "secrets_manager"},
|
|
41
|
+
{"key": "CUBEJS_DB_PASS", "description": "Data warehouse password", "destination": "secrets_manager"},
|
|
42
|
+
{"key": "CUBEJS_API_SECRET", "description": "JWT signing secret for Cube.js API \u2014 generate with: openssl rand -hex 32", "destination": "secrets_manager"},
|
|
43
|
+
{"key": "SUPERSET_URL", "description": "Superset instance URL for dbt-cube-sync", "destination": "secrets_manager"},
|
|
44
|
+
{"key": "SUPERSET_USERNAME", "description": "Superset username used by dbt-cube-sync", "destination": "secrets_manager"},
|
|
45
|
+
{"key": "SUPERSET_PASSWORD", "description": "Superset password for the sync user", "destination": "secrets_manager"}
|
|
46
|
+
],
|
|
47
|
+
"optional_secrets": [
|
|
48
|
+
{"key": "HOSTED_ZONE_ID_EC2_DNS", "description": "Route53 hosted zone ID for the EC2 DNS record (e.g. Z1MHSQZ1OR039Y)", "destination": "secrets_manager"},
|
|
49
|
+
{"key": "EC2_DNS", "description": "DNS name for the EC2 instance A record. Resolves to private IP (internal ALB) or public IP (internet-facing).", "destination": "secrets_manager"},
|
|
50
|
+
{"key": "OPENAI_API_KEY", "description": "OpenAI key for RAG embeddings", "destination": "secrets_manager"},
|
|
51
|
+
{"key": "DATABASE_URI", "description": "SQLAlchemy URI for dbt-cube-sync", "destination": "secrets_manager"},
|
|
52
|
+
{"key": "AWS_DEFAULT_REGION", "description": "AWS region (default: us-east-1)", "destination": "secrets_manager"},
|
|
53
|
+
{"key": "LLM_PROVIDER", "description": "LLM provider (openai | anthropic | bedrock | ollama)", "destination": "secrets_manager"},
|
|
54
|
+
{"key": "LLM_MODEL_ID", "description": "LLM model identifier", "destination": "secrets_manager"},
|
|
55
|
+
{"key": "LLM_API_KEY", "description": "LLM API key (if not using Bedrock)", "destination": "secrets_manager"}
|
|
56
|
+
]
|
|
57
|
+
}
|
|
58
|
+
}
|
dwe/cli.py
ADDED
|
@@ -0,0 +1,591 @@
|
|
|
1
|
+
# Copyright 2026 Ponder
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""DWE CLI — Data Warehouse Ecosystem Orchestrator.
|
|
16
|
+
|
|
17
|
+
Workflow (create-service):
|
|
18
|
+
1. Clone client repo → GitPython
|
|
19
|
+
2. Run copier.run_copy(adapter → local clone) → Copier (smart template engine)
|
|
20
|
+
3. Write dwe-state.json → CLI post-processing
|
|
21
|
+
4. Generate per-env CI/CD workflows → Jinja2 post-processing
|
|
22
|
+
5. Branch (initial-commit) + per-env branches → GitPython
|
|
23
|
+
6. Push → GitPython
|
|
24
|
+
7. Inject secrets → PyGithub / python-gitlab
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import json
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import Optional
|
|
32
|
+
|
|
33
|
+
import typer
|
|
34
|
+
from jinja2 import Environment, FileSystemLoader
|
|
35
|
+
from rich.console import Console
|
|
36
|
+
from rich.panel import Panel
|
|
37
|
+
from rich.table import Table
|
|
38
|
+
|
|
39
|
+
from dwe.git_ops import (
|
|
40
|
+
checkout_adapter_tag,
|
|
41
|
+
clone_repo,
|
|
42
|
+
commit_all,
|
|
43
|
+
create_and_checkout_branch,
|
|
44
|
+
detect_platform,
|
|
45
|
+
parse_repo_info,
|
|
46
|
+
push_branch,
|
|
47
|
+
)
|
|
48
|
+
from dwe.registry import get_adapter, get_adapter_catalog, list_adapters, load_registry
|
|
49
|
+
from dwe.secrets import (
|
|
50
|
+
delete_secret,
|
|
51
|
+
inject_secrets,
|
|
52
|
+
list_secret_keys,
|
|
53
|
+
set_secrets,
|
|
54
|
+
)
|
|
55
|
+
from dwe.state import read_state, update_state_version, write_state
|
|
56
|
+
|
|
57
|
+
app = typer.Typer(
|
|
58
|
+
name="dwe",
|
|
59
|
+
help="DWE CLI — Data Warehouse Ecosystem Orchestrator",
|
|
60
|
+
add_completion=False,
|
|
61
|
+
)
|
|
62
|
+
console = Console()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# ---------------------------------------------------------------------------
|
|
66
|
+
# Helpers
|
|
67
|
+
# ---------------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
def _resolve_adapter_version(adapter_path: str, tag: Optional[str]) -> str:
|
|
70
|
+
"""Read version from adapter.json (or copier.yml default), preferring --tag."""
|
|
71
|
+
if tag:
|
|
72
|
+
return tag
|
|
73
|
+
meta_file = Path(adapter_path) / "adapter.json"
|
|
74
|
+
if meta_file.exists():
|
|
75
|
+
return json.loads(meta_file.read_text()).get("version", "v1.0.0")
|
|
76
|
+
return "v1.0.0"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _generate_ci_workflows(
|
|
80
|
+
adapter_path: str,
|
|
81
|
+
repo_path: str,
|
|
82
|
+
environments: list[str],
|
|
83
|
+
platform: str,
|
|
84
|
+
aws_region: str,
|
|
85
|
+
) -> None:
|
|
86
|
+
"""Render ci-templates/{platform}.yaml for each environment and write to repo."""
|
|
87
|
+
ci_templates_dir = Path(adapter_path) / "ci-templates"
|
|
88
|
+
template_file = f"{platform}.yaml"
|
|
89
|
+
|
|
90
|
+
if not ci_templates_dir.exists() or not (ci_templates_dir / template_file).exists():
|
|
91
|
+
console.print(
|
|
92
|
+
f"[yellow]No ci-templates/{template_file} found in adapter, skipping CI/CD generation[/yellow]"
|
|
93
|
+
)
|
|
94
|
+
return
|
|
95
|
+
|
|
96
|
+
# Use {@ @} as variable delimiters so ${{ }} GitHub/GitLab syntax is
|
|
97
|
+
# never interpreted by Jinja2 and passes through verbatim.
|
|
98
|
+
jinja_env = Environment(
|
|
99
|
+
loader=FileSystemLoader(str(ci_templates_dir)),
|
|
100
|
+
variable_start_string="{@",
|
|
101
|
+
variable_end_string="@}",
|
|
102
|
+
keep_trailing_newline=True,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
if platform == "github":
|
|
106
|
+
workflows_dir = Path(repo_path) / ".github" / "workflows"
|
|
107
|
+
else:
|
|
108
|
+
workflows_dir = Path(repo_path) / ".gitlab" / "workflows"
|
|
109
|
+
workflows_dir.mkdir(parents=True, exist_ok=True)
|
|
110
|
+
|
|
111
|
+
template = jinja_env.get_template(template_file)
|
|
112
|
+
for env_name in environments:
|
|
113
|
+
rendered = template.render(ENV_NAME=env_name, AWS_REGION=aws_region)
|
|
114
|
+
output = workflows_dir / f"deploy-{env_name}.yaml"
|
|
115
|
+
output.write_text(rendered)
|
|
116
|
+
console.print(f"[green]CI/CD generated:[/green] {output.relative_to(repo_path)}")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# ---------------------------------------------------------------------------
|
|
120
|
+
# Commands
|
|
121
|
+
# ---------------------------------------------------------------------------
|
|
122
|
+
|
|
123
|
+
@app.command("create-service")
|
|
124
|
+
def create_service(
|
|
125
|
+
adapter_name: str = typer.Argument(..., help="Adapter name (from registry, e.g. test_adapter)"),
|
|
126
|
+
git_repo: str = typer.Option(..., "--git-repo", help="URL of the client's git repository"),
|
|
127
|
+
secrets: Optional[str] = typer.Option(
|
|
128
|
+
None, "--secrets", help='JSON string of secrets, e.g. \'{"AWS_KEY":"val"}\''
|
|
129
|
+
),
|
|
130
|
+
envs: Optional[list[str]] = typer.Option(
|
|
131
|
+
None, "--envs", help="Environment branches (repeat for multiple: --envs dev --envs prod)",
|
|
132
|
+
),
|
|
133
|
+
tag: Optional[str] = typer.Option(None, "--tag", help="Adapter version tag to use"),
|
|
134
|
+
token: Optional[str] = typer.Option(
|
|
135
|
+
None, "--token",
|
|
136
|
+
envvar=["GITHUB_TOKEN", "GITLAB_TOKEN"],
|
|
137
|
+
help="API token for secret injection",
|
|
138
|
+
),
|
|
139
|
+
aws_region: str = typer.Option("us-east-1", "--aws-region", help="AWS region"),
|
|
140
|
+
instance_type: str = typer.Option("t3.micro", "--instance-type", help="EC2 instance type"),
|
|
141
|
+
clone_dir: Optional[str] = typer.Option(
|
|
142
|
+
None, "--clone-dir", help="Directory to clone into (default: temp dir)"
|
|
143
|
+
),
|
|
144
|
+
):
|
|
145
|
+
"""Clone a client repo and inject an adapter (blueprint + infra + CI/CD)."""
|
|
146
|
+
import copier
|
|
147
|
+
|
|
148
|
+
console.print(Panel(
|
|
149
|
+
f"[bold]DWE Create Service[/bold]\n"
|
|
150
|
+
f"Adapter: [cyan]{adapter_name}[/cyan] Repo: [cyan]{git_repo}[/cyan]",
|
|
151
|
+
expand=False,
|
|
152
|
+
))
|
|
153
|
+
|
|
154
|
+
# Resolve adapter
|
|
155
|
+
adapter_info = get_adapter(adapter_name)
|
|
156
|
+
if not adapter_info:
|
|
157
|
+
console.print(f"[red]Adapter '{adapter_name}' not found.[/red] Available: {list_adapters()}")
|
|
158
|
+
raise typer.Exit(1)
|
|
159
|
+
|
|
160
|
+
adapter_path = adapter_info["path"]
|
|
161
|
+
if not Path(adapter_path).exists():
|
|
162
|
+
console.print(f"[red]Adapter path does not exist:[/red] {adapter_path}")
|
|
163
|
+
raise typer.Exit(1)
|
|
164
|
+
|
|
165
|
+
checkout_adapter_tag(adapter_path, tag)
|
|
166
|
+
adapter_version = _resolve_adapter_version(adapter_path, tag)
|
|
167
|
+
environments = envs or ["development", "main"]
|
|
168
|
+
platform = detect_platform(git_repo)
|
|
169
|
+
owner, repo_name = parse_repo_info(git_repo)
|
|
170
|
+
|
|
171
|
+
console.print(
|
|
172
|
+
f"[dim]Platform: {platform} | Owner: {owner} | Repo: {repo_name} | "
|
|
173
|
+
f"Envs: {environments} | Version: {adapter_version}[/dim]"
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
# 1. Clone client repo
|
|
177
|
+
repo, repo_path = clone_repo(git_repo, clone_dir)
|
|
178
|
+
|
|
179
|
+
# 2. Run Copier — hydrates infrastructure/ blueprint/ justfile
|
|
180
|
+
console.print("[blue]Running Copier...[/blue]")
|
|
181
|
+
copier.run_copy(
|
|
182
|
+
src_path=adapter_path,
|
|
183
|
+
dst_path=repo_path,
|
|
184
|
+
data={
|
|
185
|
+
"project_name": repo_name,
|
|
186
|
+
"adapter_name": adapter_name,
|
|
187
|
+
"adapter_version": adapter_version,
|
|
188
|
+
"environments": environments,
|
|
189
|
+
"aws_region": aws_region,
|
|
190
|
+
"instance_type": instance_type,
|
|
191
|
+
"git_platform": platform,
|
|
192
|
+
},
|
|
193
|
+
defaults=True,
|
|
194
|
+
overwrite=True,
|
|
195
|
+
unsafe=True, # allow local paths as template source
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
# 3. Write dwe-state.json (CLI-managed, not Copier-managed)
|
|
199
|
+
console.print("[blue]Writing dwe-state.json...[/blue]")
|
|
200
|
+
write_state(repo_path, adapter_name, adapter_version, environments)
|
|
201
|
+
|
|
202
|
+
# 4. Generate per-environment CI/CD workflow files
|
|
203
|
+
console.print("[blue]Generating CI/CD workflows...[/blue]")
|
|
204
|
+
_generate_ci_workflows(adapter_path, repo_path, environments, platform, aws_region)
|
|
205
|
+
|
|
206
|
+
# 5. Create initial-commit branch and commit everything
|
|
207
|
+
console.print("[blue]Creating branch: initial-commit[/blue]")
|
|
208
|
+
create_and_checkout_branch(repo, "initial-commit")
|
|
209
|
+
commit_all(repo, f"chore: inject {adapter_name} {adapter_version} via dwe")
|
|
210
|
+
|
|
211
|
+
# 6. Push initial-commit
|
|
212
|
+
push_branch(repo, "initial-commit")
|
|
213
|
+
|
|
214
|
+
# 7. Create and push per-env branches (from initial-commit)
|
|
215
|
+
for env_name in environments:
|
|
216
|
+
console.print(f"[blue]Creating branch:[/blue] {env_name}")
|
|
217
|
+
create_and_checkout_branch(repo, env_name)
|
|
218
|
+
push_branch(repo, env_name)
|
|
219
|
+
# Return to initial-commit for next env branch
|
|
220
|
+
repo.heads["initial-commit"].checkout()
|
|
221
|
+
|
|
222
|
+
# 8. Inject secrets
|
|
223
|
+
inject_secrets(git_repo, owner, repo_name, secrets, token)
|
|
224
|
+
|
|
225
|
+
# Summary
|
|
226
|
+
table = Table(title="[green]Service Created[/green]")
|
|
227
|
+
table.add_column("", style="dim")
|
|
228
|
+
table.add_column("")
|
|
229
|
+
table.add_row("Adapter", adapter_name)
|
|
230
|
+
table.add_row("Version", adapter_version)
|
|
231
|
+
table.add_row("Local path", repo_path)
|
|
232
|
+
table.add_row("Branches", ", ".join(["initial-commit", *environments]))
|
|
233
|
+
table.add_row("Infrastructure", "Pulumi")
|
|
234
|
+
table.add_row("Run", "cd " + repo_path + " && just preview")
|
|
235
|
+
console.print(table)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@app.command("update-service")
|
|
239
|
+
def update_service(
|
|
240
|
+
adapter_name: str = typer.Argument(..., help="Adapter name to update"),
|
|
241
|
+
local_path: str = typer.Argument(..., help="Local path to the client repository"),
|
|
242
|
+
tag: Optional[str] = typer.Option(None, "--tag", help="Adapter version tag to update to"),
|
|
243
|
+
):
|
|
244
|
+
"""Update an existing service with a newer adapter version (smart merge via Copier)."""
|
|
245
|
+
import copier
|
|
246
|
+
from datetime import date
|
|
247
|
+
|
|
248
|
+
console.print(Panel(
|
|
249
|
+
f"[bold]DWE Update Service[/bold]\n"
|
|
250
|
+
f"Adapter: [cyan]{adapter_name}[/cyan] Path: [cyan]{local_path}[/cyan]",
|
|
251
|
+
expand=False,
|
|
252
|
+
))
|
|
253
|
+
|
|
254
|
+
# Validate state
|
|
255
|
+
try:
|
|
256
|
+
state = read_state(local_path)
|
|
257
|
+
except FileNotFoundError as e:
|
|
258
|
+
console.print(f"[red]{e}[/red]")
|
|
259
|
+
raise typer.Exit(1)
|
|
260
|
+
|
|
261
|
+
if state["adapter"]["name"] != adapter_name:
|
|
262
|
+
console.print(
|
|
263
|
+
f"[red]Adapter mismatch:[/red] state says '{state['adapter']['name']}' "
|
|
264
|
+
f"but you specified '{adapter_name}'"
|
|
265
|
+
)
|
|
266
|
+
raise typer.Exit(1)
|
|
267
|
+
|
|
268
|
+
current_version = state["adapter"]["version"]
|
|
269
|
+
console.print(f"[dim]Current version: {current_version}[/dim]")
|
|
270
|
+
|
|
271
|
+
# Resolve adapter
|
|
272
|
+
adapter_info = get_adapter(adapter_name)
|
|
273
|
+
if not adapter_info:
|
|
274
|
+
console.print(f"[red]Adapter '{adapter_name}' not found.[/red]")
|
|
275
|
+
raise typer.Exit(1)
|
|
276
|
+
|
|
277
|
+
adapter_path = adapter_info["path"]
|
|
278
|
+
checkout_adapter_tag(adapter_path, tag)
|
|
279
|
+
new_version = _resolve_adapter_version(adapter_path, tag)
|
|
280
|
+
console.print(f"[dim]New version: {new_version}[/dim]")
|
|
281
|
+
|
|
282
|
+
# Open local repo and create update branch
|
|
283
|
+
import git as gitlib
|
|
284
|
+
try:
|
|
285
|
+
repo = gitlib.Repo(local_path)
|
|
286
|
+
except gitlib.InvalidGitRepositoryError:
|
|
287
|
+
console.print(f"[red]{local_path} is not a git repository[/red]")
|
|
288
|
+
raise typer.Exit(1)
|
|
289
|
+
|
|
290
|
+
today = date.today().strftime("%Y%m%d")
|
|
291
|
+
update_branch = f"dwe-update-{today}-{new_version.lstrip('v')}"
|
|
292
|
+
console.print(f"[blue]Creating update branch:[/blue] {update_branch}")
|
|
293
|
+
create_and_checkout_branch(repo, update_branch)
|
|
294
|
+
|
|
295
|
+
# Run Copier update — smart merge: preserves user customisations
|
|
296
|
+
console.print("[blue]Running copier.run_update (smart merge)...[/blue]")
|
|
297
|
+
copier.run_update(
|
|
298
|
+
dst_path=local_path,
|
|
299
|
+
defaults=True,
|
|
300
|
+
overwrite=True,
|
|
301
|
+
unsafe=True,
|
|
302
|
+
vcs_ref=tag,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
# Update dwe-state.json
|
|
306
|
+
update_state_version(local_path, new_version)
|
|
307
|
+
console.print(f"[green]Version updated:[/green] {current_version} → {new_version}")
|
|
308
|
+
|
|
309
|
+
# Commit update
|
|
310
|
+
commit_all(repo, f"chore: update {adapter_name} {current_version} → {new_version}")
|
|
311
|
+
|
|
312
|
+
console.print(Panel(
|
|
313
|
+
f"[green]Update branch ready:[/green] {update_branch}\n\n"
|
|
314
|
+
"Review the diff, then merge into your environment branches to trigger deployments.",
|
|
315
|
+
title="Next Steps",
|
|
316
|
+
expand=False,
|
|
317
|
+
))
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
@app.command("list-adapters")
|
|
321
|
+
def list_adapters_cmd(
|
|
322
|
+
full: bool = typer.Option(False, "--full", help="Show required secrets for each adapter"),
|
|
323
|
+
):
|
|
324
|
+
"""List all registered adapters with metadata from their copier.yml."""
|
|
325
|
+
catalog = get_adapter_catalog()
|
|
326
|
+
if not catalog:
|
|
327
|
+
console.print("[yellow]No adapters registered.[/yellow]")
|
|
328
|
+
return
|
|
329
|
+
|
|
330
|
+
table = Table(title="Registered Adapters")
|
|
331
|
+
table.add_column("Name", style="cyan")
|
|
332
|
+
table.add_column("Display Name", style="bold")
|
|
333
|
+
table.add_column("Type", style="green")
|
|
334
|
+
table.add_column("Source")
|
|
335
|
+
table.add_column("Description")
|
|
336
|
+
|
|
337
|
+
for name, info in catalog.items():
|
|
338
|
+
source = info.get("url") or info.get("path") or "N/A"
|
|
339
|
+
table.add_row(
|
|
340
|
+
name,
|
|
341
|
+
info.get("display_name", name),
|
|
342
|
+
info.get("type", "git"),
|
|
343
|
+
source,
|
|
344
|
+
info.get("description", ""),
|
|
345
|
+
)
|
|
346
|
+
console.print(table)
|
|
347
|
+
|
|
348
|
+
if full:
|
|
349
|
+
for name, info in catalog.items():
|
|
350
|
+
required = info.get("required_secrets", [])
|
|
351
|
+
optional = info.get("optional_secrets", [])
|
|
352
|
+
if not required and not optional:
|
|
353
|
+
continue
|
|
354
|
+
secrets_table = Table(title=f"[cyan]{name}[/cyan] secrets", show_header=True)
|
|
355
|
+
secrets_table.add_column("Key", style="bold")
|
|
356
|
+
secrets_table.add_column("Required")
|
|
357
|
+
secrets_table.add_column("Description")
|
|
358
|
+
for s in required:
|
|
359
|
+
secrets_table.add_row(s["key"], "[red]yes[/red]", s.get("description", ""))
|
|
360
|
+
for s in optional:
|
|
361
|
+
secrets_table.add_row(s["key"], "[dim]no[/dim]", s.get("description", ""))
|
|
362
|
+
console.print(secrets_table)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
@app.command("set-secrets")
|
|
366
|
+
def set_secrets_cmd(
|
|
367
|
+
git_repo: str = typer.Option(..., "--git-repo", help="Client repository URL"),
|
|
368
|
+
secrets: Optional[str] = typer.Option(
|
|
369
|
+
None, "--secrets", help='JSON string, e.g. \'{"AWS_KEY":"val"}\''
|
|
370
|
+
),
|
|
371
|
+
secrets_file: Optional[str] = typer.Option(
|
|
372
|
+
None, "--secrets-file", help="Path to a JSON file with secrets"
|
|
373
|
+
),
|
|
374
|
+
adapter_name: Optional[str] = typer.Option(
|
|
375
|
+
None, "--adapter", help="Validate against adapter required keys before pushing"
|
|
376
|
+
),
|
|
377
|
+
token: Optional[str] = typer.Option(
|
|
378
|
+
None, "--token",
|
|
379
|
+
envvar=["GITHUB_TOKEN", "GITLAB_TOKEN"],
|
|
380
|
+
help="API token for secret injection",
|
|
381
|
+
),
|
|
382
|
+
):
|
|
383
|
+
"""Create or update secrets / CI variables in a GitHub or GitLab repository."""
|
|
384
|
+
if not token:
|
|
385
|
+
console.print("[red]--token is required (or set GITHUB_TOKEN / GITLAB_TOKEN)[/red]")
|
|
386
|
+
raise typer.Exit(1)
|
|
387
|
+
|
|
388
|
+
if secrets_file:
|
|
389
|
+
import pathlib
|
|
390
|
+
raw = pathlib.Path(secrets_file).read_text()
|
|
391
|
+
elif secrets:
|
|
392
|
+
raw = secrets
|
|
393
|
+
else:
|
|
394
|
+
console.print("[red]Provide --secrets or --secrets-file[/red]")
|
|
395
|
+
raise typer.Exit(1)
|
|
396
|
+
|
|
397
|
+
try:
|
|
398
|
+
secrets_dict = json.loads(raw)
|
|
399
|
+
except json.JSONDecodeError as e:
|
|
400
|
+
console.print(f"[red]Invalid JSON:[/red] {e}")
|
|
401
|
+
raise typer.Exit(1)
|
|
402
|
+
|
|
403
|
+
# Optional: validate required keys before pushing
|
|
404
|
+
if adapter_name:
|
|
405
|
+
catalog = get_adapter_catalog()
|
|
406
|
+
info = catalog.get(adapter_name, {})
|
|
407
|
+
required = [s["key"] for s in info.get("required_secrets", [])]
|
|
408
|
+
missing = [k for k in required if k not in secrets_dict]
|
|
409
|
+
if missing:
|
|
410
|
+
console.print(f"[yellow]Warning — missing required keys:[/yellow] {', '.join(missing)}")
|
|
411
|
+
|
|
412
|
+
owner, repo_name = parse_repo_info(git_repo)
|
|
413
|
+
platform = "gitlab" if "gitlab" in git_repo.lower() else "github"
|
|
414
|
+
console.print(
|
|
415
|
+
f"\nPushing [bold]{len(secrets_dict)}[/bold] secret(s) to "
|
|
416
|
+
f"[cyan]{owner}/{repo_name}[/cyan] ({platform})\n"
|
|
417
|
+
)
|
|
418
|
+
results = set_secrets(git_repo, owner, repo_name, secrets_dict, token)
|
|
419
|
+
ok = sum(1 for v in results.values() if v)
|
|
420
|
+
console.print(f"\n[bold]{ok}/{len(results)}[/bold] secret(s) pushed successfully.")
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
@app.command("list-secrets")
|
|
424
|
+
def list_secrets_cmd(
|
|
425
|
+
git_repo: str = typer.Option(..., "--git-repo", help="Client repository URL"),
|
|
426
|
+
adapter_name: Optional[str] = typer.Option(
|
|
427
|
+
None, "--adapter", help="Cross-reference keys against adapter requirements"
|
|
428
|
+
),
|
|
429
|
+
token: Optional[str] = typer.Option(
|
|
430
|
+
None, "--token",
|
|
431
|
+
envvar=["GITHUB_TOKEN", "GITLAB_TOKEN"],
|
|
432
|
+
),
|
|
433
|
+
):
|
|
434
|
+
"""List secret key names in a repository (values are never revealed)."""
|
|
435
|
+
if not token:
|
|
436
|
+
console.print("[red]--token is required (or set GITHUB_TOKEN / GITLAB_TOKEN)[/red]")
|
|
437
|
+
raise typer.Exit(1)
|
|
438
|
+
|
|
439
|
+
owner, repo_name = parse_repo_info(git_repo)
|
|
440
|
+
existing = set(list_secret_keys(git_repo, owner, repo_name, token))
|
|
441
|
+
|
|
442
|
+
required: list[str] = []
|
|
443
|
+
optional: list[str] = []
|
|
444
|
+
if adapter_name:
|
|
445
|
+
catalog = get_adapter_catalog()
|
|
446
|
+
info = catalog.get(adapter_name, {})
|
|
447
|
+
required = [s["key"] for s in info.get("required_secrets", [])]
|
|
448
|
+
optional = [s["key"] for s in info.get("optional_secrets", [])]
|
|
449
|
+
|
|
450
|
+
all_keys = sorted(existing | set(required) | set(optional))
|
|
451
|
+
|
|
452
|
+
table = Table(title=f"Secrets — [cyan]{owner}/{repo_name}[/cyan]")
|
|
453
|
+
table.add_column("Key")
|
|
454
|
+
table.add_column("Set", justify="center")
|
|
455
|
+
if adapter_name:
|
|
456
|
+
table.add_column("Required", justify="center")
|
|
457
|
+
|
|
458
|
+
for key in all_keys:
|
|
459
|
+
is_set = "[green]✓[/green]" if key in existing else "[red]✗[/red]"
|
|
460
|
+
if adapter_name:
|
|
461
|
+
if key in required:
|
|
462
|
+
req_col = "[red]yes[/red]"
|
|
463
|
+
elif key in optional:
|
|
464
|
+
req_col = "[dim]optional[/dim]"
|
|
465
|
+
else:
|
|
466
|
+
req_col = ""
|
|
467
|
+
table.add_row(key, is_set, req_col)
|
|
468
|
+
else:
|
|
469
|
+
table.add_row(key, is_set)
|
|
470
|
+
console.print(table)
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
@app.command("delete-secret")
|
|
474
|
+
def delete_secret_cmd(
|
|
475
|
+
git_repo: str = typer.Option(..., "--git-repo", help="Client repository URL"),
|
|
476
|
+
key: str = typer.Option(..., "--key", help="Secret / variable key to delete"),
|
|
477
|
+
token: Optional[str] = typer.Option(
|
|
478
|
+
None, "--token",
|
|
479
|
+
envvar=["GITHUB_TOKEN", "GITLAB_TOKEN"],
|
|
480
|
+
),
|
|
481
|
+
):
|
|
482
|
+
"""Delete a single secret / CI variable from a repository."""
|
|
483
|
+
if not token:
|
|
484
|
+
console.print("[red]--token is required (or set GITHUB_TOKEN / GITLAB_TOKEN)[/red]")
|
|
485
|
+
raise typer.Exit(1)
|
|
486
|
+
|
|
487
|
+
owner, repo_name = parse_repo_info(git_repo)
|
|
488
|
+
delete_secret(git_repo, owner, repo_name, key, token)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
@app.command("show-properties")
|
|
492
|
+
def show_properties(
|
|
493
|
+
adapter_name: str = typer.Argument(..., help="Adapter name (as in adapters.json)"),
|
|
494
|
+
):
|
|
495
|
+
"""Show git providers, cloud providers, services, and CI templates for an adapter."""
|
|
496
|
+
catalog = get_adapter_catalog()
|
|
497
|
+
if adapter_name not in catalog:
|
|
498
|
+
console.print(f"[red]Adapter '{adapter_name}' not found.[/red] Available: {list(catalog)}")
|
|
499
|
+
raise typer.Exit(1)
|
|
500
|
+
|
|
501
|
+
info = catalog[adapter_name]
|
|
502
|
+
|
|
503
|
+
props = Table(title=f"[cyan]{adapter_name}[/cyan] properties", show_header=False, box=None)
|
|
504
|
+
props.add_column("Property", style="dim", min_width=20)
|
|
505
|
+
props.add_column("Value")
|
|
506
|
+
props.add_row("Display name", info.get("display_name", ""))
|
|
507
|
+
props.add_row("Description", info.get("description", ""))
|
|
508
|
+
props.add_row("Hub name", info.get("hub_name", ""))
|
|
509
|
+
props.add_row("Source URL", info.get("url") or info.get("path", ""))
|
|
510
|
+
props.add_row("Type", info.get("type", ""))
|
|
511
|
+
props.add_row("Git providers", ", ".join(info.get("git_providers", [])) or "—")
|
|
512
|
+
props.add_row("Cloud providers", ", ".join(info.get("cloud_providers", [])) or "—")
|
|
513
|
+
|
|
514
|
+
ci = info.get("ci_templates", {})
|
|
515
|
+
props.add_row("CI templates", ", ".join(ci.keys()) if ci else "—")
|
|
516
|
+
console.print(props)
|
|
517
|
+
|
|
518
|
+
services = info.get("services", [])
|
|
519
|
+
if services:
|
|
520
|
+
svc_table = Table(title="Services", show_header=True)
|
|
521
|
+
svc_table.add_column("Service", style="cyan")
|
|
522
|
+
svc_table.add_column("Description")
|
|
523
|
+
for s in services:
|
|
524
|
+
svc_table.add_row(s.get("name", ""), s.get("description", ""))
|
|
525
|
+
console.print(svc_table)
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
@app.command("show-services")
|
|
529
|
+
def show_services(
|
|
530
|
+
adapter_name: str = typer.Argument(..., help="Adapter name (as in adapters.json)"),
|
|
531
|
+
):
|
|
532
|
+
"""List the services bundled in an adapter."""
|
|
533
|
+
catalog = get_adapter_catalog()
|
|
534
|
+
if adapter_name not in catalog:
|
|
535
|
+
console.print(f"[red]Adapter '{adapter_name}' not found.[/red] Available: {list(catalog)}")
|
|
536
|
+
raise typer.Exit(1)
|
|
537
|
+
|
|
538
|
+
services = catalog[adapter_name].get("services", [])
|
|
539
|
+
if not services:
|
|
540
|
+
console.print(f"[yellow]No services defined for '{adapter_name}'.[/yellow]")
|
|
541
|
+
return
|
|
542
|
+
|
|
543
|
+
table = Table(title=f"[cyan]{adapter_name}[/cyan] services")
|
|
544
|
+
table.add_column("Service", style="cyan bold")
|
|
545
|
+
table.add_column("Description")
|
|
546
|
+
for s in services:
|
|
547
|
+
table.add_row(s.get("name", ""), s.get("description", ""))
|
|
548
|
+
console.print(table)
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
@app.command("show-secrets-template")
|
|
552
|
+
def show_secrets_template(
|
|
553
|
+
adapter_name: str = typer.Argument(..., help="Adapter name (as in adapters.json)"),
|
|
554
|
+
cloud_provider: str = typer.Option("aws", "--cloud", help="Cloud provider filter (aws)"),
|
|
555
|
+
git_provider: str = typer.Option("github", "--git-provider", help="Git provider (github | gitlab)"),
|
|
556
|
+
):
|
|
557
|
+
"""Print a JSON secrets template for the adapter — copy, fill values, upload to AWS/GitHub."""
|
|
558
|
+
import json as _json
|
|
559
|
+
|
|
560
|
+
catalog = get_adapter_catalog()
|
|
561
|
+
if adapter_name not in catalog:
|
|
562
|
+
console.print(f"[red]Adapter '{adapter_name}' not found.[/red] Available: {list(catalog)}")
|
|
563
|
+
raise typer.Exit(1)
|
|
564
|
+
|
|
565
|
+
info = catalog[adapter_name]
|
|
566
|
+
required = info.get("required_secrets", [])
|
|
567
|
+
optional = info.get("optional_secrets", [])
|
|
568
|
+
|
|
569
|
+
template: dict = {}
|
|
570
|
+
for s in required:
|
|
571
|
+
template[s["key"]] = ""
|
|
572
|
+
for s in optional:
|
|
573
|
+
template[s["key"]] = ""
|
|
574
|
+
|
|
575
|
+
console.print(f"\n[bold]Secrets template for[/bold] [cyan]{adapter_name}[/cyan] "
|
|
576
|
+
f"(cloud: {cloud_provider}, git: {git_provider})\n")
|
|
577
|
+
console.print(_json.dumps(template, indent=2))
|
|
578
|
+
|
|
579
|
+
if required:
|
|
580
|
+
console.print("\n[bold red]Required keys:[/bold red]")
|
|
581
|
+
for s in required:
|
|
582
|
+
console.print(f" [red]•[/red] [bold]{s['key']}[/bold] — {s.get('description', '')}")
|
|
583
|
+
|
|
584
|
+
if optional:
|
|
585
|
+
console.print("\n[dim]Optional keys:[/dim]")
|
|
586
|
+
for s in optional:
|
|
587
|
+
console.print(f" [dim]•[/dim] {s['key']} — {s.get('description', '')}")
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
if __name__ == "__main__":
|
|
591
|
+
app()
|