steve-cli 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {steve_cli-0.1.0 → steve_cli-0.3.0}/PKG-INFO +30 -3
- {steve_cli-0.1.0 → steve_cli-0.3.0}/README.md +22 -3
- {steve_cli-0.1.0 → steve_cli-0.3.0}/pyproject.toml +20 -2
- {steve_cli-0.1.0 → steve_cli-0.3.0}/steve_cli/cli.py +128 -97
- steve_cli-0.3.0/steve_cli/storage.py +158 -0
- {steve_cli-0.1.0 → steve_cli-0.3.0}/steve_cli.egg-info/PKG-INFO +30 -3
- {steve_cli-0.1.0 → steve_cli-0.3.0}/steve_cli.egg-info/SOURCES.txt +1 -0
- steve_cli-0.3.0/steve_cli.egg-info/requires.txt +17 -0
- steve_cli-0.1.0/steve_cli.egg-info/requires.txt +0 -7
- {steve_cli-0.1.0 → steve_cli-0.3.0}/setup.cfg +0 -0
- {steve_cli-0.1.0 → steve_cli-0.3.0}/steve_cli/__init__.py +0 -0
- {steve_cli-0.1.0 → steve_cli-0.3.0}/steve_cli.egg-info/dependency_links.txt +0 -0
- {steve_cli-0.1.0 → steve_cli-0.3.0}/steve_cli.egg-info/entry_points.txt +0 -0
- {steve_cli-0.1.0 → steve_cli-0.3.0}/steve_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steve-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
|
|
5
5
|
Author: Frank
|
|
6
6
|
License: MIT
|
|
@@ -17,12 +17,20 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
18
|
Requires-Python: >=3.8
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
|
+
Requires-Dist: boto3>=1.37.38
|
|
20
21
|
Requires-Dist: click>=8.0.0
|
|
21
22
|
Requires-Dist: pyyaml>=6.0
|
|
23
|
+
Requires-Dist: questionary>=2.0.0
|
|
22
24
|
Provides-Extra: dev
|
|
23
25
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
24
26
|
Requires-Dist: black>=22.0; extra == "dev"
|
|
25
27
|
Requires-Dist: isort>=5.0; extra == "dev"
|
|
28
|
+
Provides-Extra: polars
|
|
29
|
+
Requires-Dist: polars>=1.8.2; extra == "polars"
|
|
30
|
+
Requires-Dist: pyarrow>=17.0.0; extra == "polars"
|
|
31
|
+
Provides-Extra: pandas
|
|
32
|
+
Requires-Dist: pandas>=2.0.3; extra == "pandas"
|
|
33
|
+
Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
|
|
26
34
|
|
|
27
35
|
# Steve CLI
|
|
28
36
|
|
|
@@ -40,21 +48,28 @@ uv pip install -e .
|
|
|
40
48
|
|
|
41
49
|
### Install from GitHub
|
|
42
50
|
|
|
51
|
+
`uv add steve-cli`
|
|
52
|
+
|
|
53
|
+
OR
|
|
54
|
+
|
|
43
55
|
```bash
|
|
44
56
|
uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
|
|
45
57
|
```
|
|
46
58
|
|
|
59
|
+
|
|
60
|
+
|
|
47
61
|
## Usage
|
|
48
62
|
|
|
49
63
|
### Run a job
|
|
50
64
|
|
|
51
65
|
```bash
|
|
52
66
|
steve extract-data
|
|
53
|
-
steve transform-data
|
|
67
|
+
steve transform-data
|
|
54
68
|
steve validate-data
|
|
55
69
|
```
|
|
56
70
|
|
|
57
71
|
This will:
|
|
72
|
+
|
|
58
73
|
1. Look for `jobs.yaml` in the current directory
|
|
59
74
|
2. Find the specified job by name
|
|
60
75
|
3. Set all environment variables from the job's `env` section
|
|
@@ -94,7 +109,7 @@ jobs:
|
|
|
94
109
|
env:
|
|
95
110
|
SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
|
|
96
111
|
OUTPUT_PATH: "/data/raw/extracted_data.parquet"
|
|
97
|
-
|
|
112
|
+
|
|
98
113
|
- name: transform-data
|
|
99
114
|
dependsOn: ["extract-data"]
|
|
100
115
|
command: ["uv", "run", "src/transform.py"]
|
|
@@ -104,6 +119,7 @@ jobs:
|
|
|
104
119
|
```
|
|
105
120
|
|
|
106
121
|
Running `steve extract-data` will:
|
|
122
|
+
|
|
107
123
|
1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
|
|
108
124
|
2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
|
|
109
125
|
3. Execute: `uv run src/extract.py`
|
|
@@ -141,3 +157,14 @@ pytest
|
|
|
141
157
|
black steve_cli/
|
|
142
158
|
isort steve_cli/
|
|
143
159
|
```
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
# use cli
|
|
164
|
+
source .venv/bin/activate
|
|
165
|
+
steve
|
|
166
|
+
|
|
167
|
+
#
|
|
168
|
+
|
|
169
|
+
uv build
|
|
170
|
+
uv publish
|
|
@@ -14,21 +14,28 @@ uv pip install -e .
|
|
|
14
14
|
|
|
15
15
|
### Install from GitHub
|
|
16
16
|
|
|
17
|
+
`uv add steve-cli`
|
|
18
|
+
|
|
19
|
+
OR
|
|
20
|
+
|
|
17
21
|
```bash
|
|
18
22
|
uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
|
|
19
23
|
```
|
|
20
24
|
|
|
25
|
+
|
|
26
|
+
|
|
21
27
|
## Usage
|
|
22
28
|
|
|
23
29
|
### Run a job
|
|
24
30
|
|
|
25
31
|
```bash
|
|
26
32
|
steve extract-data
|
|
27
|
-
steve transform-data
|
|
33
|
+
steve transform-data
|
|
28
34
|
steve validate-data
|
|
29
35
|
```
|
|
30
36
|
|
|
31
37
|
This will:
|
|
38
|
+
|
|
32
39
|
1. Look for `jobs.yaml` in the current directory
|
|
33
40
|
2. Find the specified job by name
|
|
34
41
|
3. Set all environment variables from the job's `env` section
|
|
@@ -68,7 +75,7 @@ jobs:
|
|
|
68
75
|
env:
|
|
69
76
|
SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
|
|
70
77
|
OUTPUT_PATH: "/data/raw/extracted_data.parquet"
|
|
71
|
-
|
|
78
|
+
|
|
72
79
|
- name: transform-data
|
|
73
80
|
dependsOn: ["extract-data"]
|
|
74
81
|
command: ["uv", "run", "src/transform.py"]
|
|
@@ -78,6 +85,7 @@ jobs:
|
|
|
78
85
|
```
|
|
79
86
|
|
|
80
87
|
Running `steve extract-data` will:
|
|
88
|
+
|
|
81
89
|
1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
|
|
82
90
|
2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
|
|
83
91
|
3. Execute: `uv run src/extract.py`
|
|
@@ -114,4 +122,15 @@ pytest
|
|
|
114
122
|
# Format code
|
|
115
123
|
black steve_cli/
|
|
116
124
|
isort steve_cli/
|
|
117
|
-
```
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# use cli
|
|
130
|
+
source .venv/bin/activate
|
|
131
|
+
steve
|
|
132
|
+
|
|
133
|
+
#
|
|
134
|
+
|
|
135
|
+
uv build
|
|
136
|
+
uv publish
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
|
|
2
|
+
[build-system]
|
|
3
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
4
|
+
build-backend = "setuptools.build_meta"
|
|
5
|
+
|
|
2
6
|
[project]
|
|
3
7
|
name = "steve-cli"
|
|
4
|
-
version = "0.
|
|
8
|
+
version = "0.3.0"
|
|
5
9
|
description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
|
|
6
10
|
readme = "README.md"
|
|
7
11
|
license = {text = "MIT"}
|
|
@@ -19,8 +23,10 @@ classifiers = [
|
|
|
19
23
|
]
|
|
20
24
|
requires-python = ">=3.8"
|
|
21
25
|
dependencies = [
|
|
26
|
+
"boto3>=1.37.38",
|
|
22
27
|
"click>=8.0.0",
|
|
23
28
|
"pyyaml>=6.0",
|
|
29
|
+
"questionary>=2.0.0",
|
|
24
30
|
]
|
|
25
31
|
|
|
26
32
|
[project.optional-dependencies]
|
|
@@ -29,6 +35,14 @@ dev = [
|
|
|
29
35
|
"black>=22.0",
|
|
30
36
|
"isort>=5.0",
|
|
31
37
|
]
|
|
38
|
+
polars = [
|
|
39
|
+
"polars>=1.8.2",
|
|
40
|
+
"pyarrow>=17.0.0",
|
|
41
|
+
]
|
|
42
|
+
pandas = [
|
|
43
|
+
"pandas>=2.0.3",
|
|
44
|
+
"pyarrow>=17.0.0",
|
|
45
|
+
]
|
|
32
46
|
|
|
33
47
|
[project.scripts]
|
|
34
48
|
steve = "steve_cli.cli:main"
|
|
@@ -39,9 +53,13 @@ Repository = "https://github.com/7frank/ds-steve-cli"
|
|
|
39
53
|
|
|
40
54
|
|
|
41
55
|
|
|
56
|
+
[tool.setuptools.packages.find]
|
|
57
|
+
where = ["."]
|
|
58
|
+
include = ["steve_cli*"]
|
|
59
|
+
|
|
42
60
|
[tool.black]
|
|
43
61
|
line-length = 88
|
|
44
62
|
target-version = ['py38']
|
|
45
63
|
|
|
46
64
|
[tool.isort]
|
|
47
|
-
profile = "black"
|
|
65
|
+
profile = "black"
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
Steve CLI - A simple tool to run jobs from jobs.yaml with proper environment setup.
|
|
4
4
|
|
|
5
5
|
Usage:
|
|
6
|
-
steve
|
|
7
|
-
steve
|
|
8
|
-
steve
|
|
6
|
+
steve jobs ls List all available jobs
|
|
7
|
+
steve jobs run <job-name> Run a job with its environment variables
|
|
8
|
+
steve setup all Decrypt SOPS-encoded .env files
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import os
|
|
@@ -15,90 +15,74 @@ from pathlib import Path
|
|
|
15
15
|
from typing import Dict, List, Any, Optional
|
|
16
16
|
|
|
17
17
|
import click
|
|
18
|
+
import questionary
|
|
18
19
|
import yaml
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
class JobsConfig:
|
|
22
|
-
"""Handle parsing and querying of jobs.yaml configuration."""
|
|
23
|
-
|
|
24
23
|
def __init__(self, jobs_file: Optional[Path] = None):
|
|
25
24
|
self.jobs_file = jobs_file or Path.cwd() / "jobs.yaml"
|
|
26
25
|
self.jobs: List[Dict[str, Any]] = []
|
|
27
26
|
self._load_jobs()
|
|
28
|
-
|
|
27
|
+
|
|
29
28
|
def _load_jobs(self) -> None:
|
|
30
|
-
"""Load and parse the jobs.yaml file."""
|
|
31
29
|
try:
|
|
32
30
|
if not self.jobs_file.exists():
|
|
33
31
|
click.echo(f"❌ Error: jobs.yaml not found at {self.jobs_file}", err=True)
|
|
34
32
|
click.echo("Make sure you're in a directory with a jobs.yaml file", err=True)
|
|
35
33
|
sys.exit(1)
|
|
36
|
-
|
|
34
|
+
|
|
37
35
|
with open(self.jobs_file, 'r') as f:
|
|
38
36
|
data = yaml.safe_load(f)
|
|
39
|
-
|
|
37
|
+
|
|
40
38
|
if not data or 'jobs' not in data:
|
|
41
|
-
click.echo(
|
|
39
|
+
click.echo("❌ Error: Invalid jobs.yaml format. Expected 'jobs' key at root", err=True)
|
|
42
40
|
sys.exit(1)
|
|
43
|
-
|
|
41
|
+
|
|
44
42
|
self.jobs = data['jobs']
|
|
45
|
-
|
|
43
|
+
|
|
46
44
|
except yaml.YAMLError as e:
|
|
47
45
|
click.echo(f"❌ Error parsing jobs.yaml: {e}", err=True)
|
|
48
46
|
sys.exit(1)
|
|
49
47
|
except Exception as e:
|
|
50
48
|
click.echo(f"❌ Error reading jobs.yaml: {e}", err=True)
|
|
51
49
|
sys.exit(1)
|
|
52
|
-
|
|
50
|
+
|
|
53
51
|
def get_job(self, job_name: str) -> Optional[Dict[str, Any]]:
|
|
54
|
-
"""Get a job by name."""
|
|
55
52
|
for job in self.jobs:
|
|
56
53
|
if job.get('name') == job_name:
|
|
57
54
|
return job
|
|
58
55
|
return None
|
|
59
|
-
|
|
56
|
+
|
|
60
57
|
def list_jobs(self) -> List[str]:
|
|
61
|
-
"""Get list of all job names."""
|
|
62
58
|
return [job.get('name', 'unnamed') for job in self.jobs]
|
|
63
59
|
|
|
64
60
|
|
|
65
61
|
def run_job_command(job: Dict[str, Any]) -> int:
|
|
66
|
-
"""Run a job's command with its environment variables."""
|
|
67
|
-
# Get command
|
|
68
62
|
command = job.get('command', [])
|
|
69
63
|
if not command:
|
|
70
64
|
click.echo(f"❌ Error: No command specified for job '{job.get('name')}'", err=True)
|
|
71
65
|
return 1
|
|
72
|
-
|
|
73
|
-
# Prepare environment
|
|
66
|
+
|
|
74
67
|
env = os.environ.copy()
|
|
75
68
|
job_env = job.get('env', {})
|
|
76
|
-
|
|
77
|
-
# Convert all env values to strings and merge
|
|
78
69
|
for key, value in job_env.items():
|
|
79
70
|
env[key] = str(value)
|
|
80
|
-
|
|
81
|
-
# Show what we're running
|
|
71
|
+
|
|
82
72
|
job_name = job.get('name', 'unnamed')
|
|
83
73
|
click.echo(f"🚀 Running job: {click.style(job_name, fg='blue', bold=True)}")
|
|
84
74
|
click.echo(f"📝 Command: {click.style(' '.join(command), fg='cyan')}")
|
|
85
|
-
|
|
75
|
+
|
|
86
76
|
if job_env:
|
|
87
|
-
click.echo(
|
|
77
|
+
click.echo("🌍 Environment variables:")
|
|
88
78
|
for key, value in job_env.items():
|
|
89
79
|
click.echo(f" {click.style(key, fg='green')}={click.style(str(value), fg='yellow')}")
|
|
90
|
-
|
|
91
|
-
click.echo()
|
|
92
|
-
|
|
80
|
+
|
|
81
|
+
click.echo()
|
|
82
|
+
|
|
93
83
|
try:
|
|
94
|
-
|
|
95
|
-
result = subprocess.run(
|
|
96
|
-
command,
|
|
97
|
-
env=env,
|
|
98
|
-
cwd=Path.cwd(),
|
|
99
|
-
)
|
|
84
|
+
result = subprocess.run(command, env=env, cwd=Path.cwd())
|
|
100
85
|
return result.returncode
|
|
101
|
-
|
|
102
86
|
except FileNotFoundError:
|
|
103
87
|
click.echo(f"❌ Error: Command not found: {command[0]}", err=True)
|
|
104
88
|
return 127
|
|
@@ -107,100 +91,147 @@ def run_job_command(job: Dict[str, Any]) -> int:
|
|
|
107
91
|
return 1
|
|
108
92
|
|
|
109
93
|
|
|
110
|
-
@click.group(
|
|
111
|
-
|
|
94
|
+
@click.group()
|
|
95
|
+
def main():
|
|
96
|
+
"""Steve CLI - Run jobs and manage environment setup."""
|
|
97
|
+
pass
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@main.group(invoke_without_command=True)
|
|
101
|
+
@click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
|
|
112
102
|
help='Path to jobs.yaml file (default: ./jobs.yaml)')
|
|
113
|
-
@click.argument('job_name', required=False)
|
|
114
103
|
@click.pass_context
|
|
115
|
-
def
|
|
116
|
-
"""
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
list_jobs_command(config)
|
|
125
|
-
return
|
|
126
|
-
elif job_name in ['help', '--help', '-h']:
|
|
127
|
-
click.echo(ctx.get_help())
|
|
128
|
-
return
|
|
129
|
-
|
|
130
|
-
# Try to run the job
|
|
131
|
-
job = config.get_job(job_name)
|
|
132
|
-
if not job:
|
|
133
|
-
available_jobs = config.list_jobs()
|
|
134
|
-
click.echo(f"❌ Error: Job '{job_name}' not found", err=True)
|
|
135
|
-
if available_jobs:
|
|
136
|
-
click.echo(f"Available jobs: {', '.join(available_jobs)}", err=True)
|
|
137
|
-
else:
|
|
138
|
-
click.echo("No jobs found in jobs.yaml", err=True)
|
|
139
|
-
sys.exit(1)
|
|
140
|
-
|
|
141
|
-
exit_code = run_job_command(job)
|
|
142
|
-
sys.exit(exit_code)
|
|
143
|
-
|
|
144
|
-
# If no job name provided, show help
|
|
104
|
+
def jobs(ctx: click.Context, jobs_file: Optional[Path]):
|
|
105
|
+
"""Manage and run jobs from jobs.yaml."""
|
|
106
|
+
if ctx.invoked_subcommand is not None:
|
|
107
|
+
return
|
|
108
|
+
config = JobsConfig(jobs_file)
|
|
109
|
+
job_names = config.list_jobs()
|
|
110
|
+
if not job_names:
|
|
111
|
+
click.echo("❌ No jobs found in jobs.yaml")
|
|
112
|
+
return
|
|
145
113
|
click.echo(ctx.get_help())
|
|
114
|
+
click.echo()
|
|
115
|
+
choice = questionary.select(
|
|
116
|
+
"Select a job to run:",
|
|
117
|
+
choices=job_names,
|
|
118
|
+
).ask()
|
|
119
|
+
if choice is None:
|
|
120
|
+
sys.exit(0)
|
|
121
|
+
job = config.get_job(choice)
|
|
122
|
+
sys.exit(run_job_command(job))
|
|
146
123
|
|
|
147
124
|
|
|
148
|
-
@
|
|
125
|
+
@jobs.command('ls')
|
|
149
126
|
@click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
|
|
150
127
|
help='Path to jobs.yaml file (default: ./jobs.yaml)')
|
|
151
|
-
def
|
|
128
|
+
def jobs_list(jobs_file: Optional[Path]):
|
|
152
129
|
"""List all available jobs."""
|
|
153
130
|
config = JobsConfig(jobs_file)
|
|
154
|
-
|
|
155
|
-
|
|
131
|
+
job_names = config.list_jobs()
|
|
156
132
|
|
|
157
|
-
|
|
158
|
-
"""Show list of available jobs with details."""
|
|
159
|
-
jobs = config.list_jobs()
|
|
160
|
-
|
|
161
|
-
if not jobs:
|
|
133
|
+
if not job_names:
|
|
162
134
|
click.echo("❌ No jobs found in jobs.yaml")
|
|
163
135
|
return
|
|
164
|
-
|
|
136
|
+
|
|
165
137
|
click.echo(f"📋 Available jobs in {click.style(str(config.jobs_file), fg='cyan')}:")
|
|
166
138
|
click.echo()
|
|
167
|
-
|
|
139
|
+
|
|
168
140
|
for job_data in config.jobs:
|
|
169
141
|
name = job_data.get('name', 'unnamed')
|
|
170
142
|
command = job_data.get('command', [])
|
|
171
143
|
cron = job_data.get('cron')
|
|
172
144
|
depends_on = job_data.get('dependsOn', [])
|
|
173
145
|
env_vars = job_data.get('env', {})
|
|
174
|
-
|
|
175
|
-
# Job name
|
|
146
|
+
|
|
176
147
|
click.echo(f" {click.style(name, fg='blue', bold=True)}")
|
|
177
|
-
|
|
178
|
-
# Command
|
|
179
148
|
if command:
|
|
180
149
|
click.echo(f" Command: {click.style(' '.join(command), fg='cyan')}")
|
|
181
|
-
|
|
182
|
-
# Schedule
|
|
183
150
|
if cron:
|
|
184
151
|
click.echo(f" Schedule: {click.style(cron, fg='yellow')}")
|
|
185
|
-
|
|
186
|
-
# Dependencies
|
|
187
152
|
if depends_on:
|
|
188
153
|
click.echo(f" Depends on: {click.style(', '.join(depends_on), fg='magenta')}")
|
|
189
|
-
|
|
190
|
-
# Environment variables
|
|
191
154
|
if env_vars:
|
|
192
155
|
click.echo(f" Environment: {click.style(f'{len(env_vars)} variables', fg='green')}")
|
|
193
156
|
for key, value in env_vars.items():
|
|
194
157
|
click.echo(f" {key}={value}")
|
|
195
|
-
|
|
196
|
-
|
|
158
|
+
click.echo()
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@jobs.command('run')
|
|
162
|
+
@click.argument('job_name')
|
|
163
|
+
@click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
|
|
164
|
+
help='Path to jobs.yaml file (default: ./jobs.yaml)')
|
|
165
|
+
def jobs_run(job_name: str, jobs_file: Optional[Path]):
|
|
166
|
+
"""Run a job by name."""
|
|
167
|
+
config = JobsConfig(jobs_file)
|
|
168
|
+
job = config.get_job(job_name)
|
|
169
|
+
if not job:
|
|
170
|
+
available = config.list_jobs()
|
|
171
|
+
click.echo(f"❌ Error: Job '{job_name}' not found", err=True)
|
|
172
|
+
if available:
|
|
173
|
+
click.echo(f"Available jobs: {', '.join(available)}", err=True)
|
|
174
|
+
sys.exit(1)
|
|
175
|
+
sys.exit(run_job_command(job))
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
SETUP_IGNORE = [".env.example", ".env.template"]
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@main.group()
|
|
182
|
+
def setup():
|
|
183
|
+
"""Setup commands for environment configuration."""
|
|
184
|
+
pass
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@setup.command("env")
|
|
188
|
+
def setup_env():
|
|
189
|
+
"""Decrypt SOPS-encoded .env files in current directory and write plaintext .env files."""
|
|
190
|
+
cwd = Path.cwd()
|
|
191
|
+
found: List[Path] = []
|
|
192
|
+
for pattern in ["*.enc.env", "*.encoded.env"]:
|
|
193
|
+
found.extend(sorted(cwd.glob(pattern)))
|
|
194
|
+
|
|
195
|
+
found = [f for f in found if f.name not in SETUP_IGNORE]
|
|
196
|
+
|
|
197
|
+
if not found:
|
|
198
|
+
click.echo("No *.enc.env or *.encoded.env files found.")
|
|
199
|
+
return
|
|
200
|
+
|
|
201
|
+
for enc_file in found:
|
|
202
|
+
click.echo(f"\n📄 Found: {click.style(enc_file.name, fg='blue', bold=True)}")
|
|
203
|
+
|
|
204
|
+
content = enc_file.read_text()
|
|
205
|
+
if "ENC[" not in content and "sops:" not in content:
|
|
206
|
+
click.echo(" ⚠️ Not SOPS encrypted, skipping.")
|
|
207
|
+
continue
|
|
208
|
+
|
|
209
|
+
result = subprocess.run(
|
|
210
|
+
["sops", "-d", str(enc_file)],
|
|
211
|
+
capture_output=True,
|
|
212
|
+
text=True,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
if result.returncode != 0:
|
|
216
|
+
click.echo(f" ❌ Decryption failed: {result.stderr.strip()}", err=True)
|
|
217
|
+
continue
|
|
218
|
+
|
|
219
|
+
keys = []
|
|
220
|
+
for line in result.stdout.splitlines():
|
|
221
|
+
line = line.strip()
|
|
222
|
+
if line and not line.startswith("#") and "=" in line:
|
|
223
|
+
keys.append(line.split("=", 1)[0])
|
|
224
|
+
|
|
225
|
+
if keys:
|
|
226
|
+
click.echo(f" 🔑 Keys: {click.style(', '.join(keys), fg='green')}")
|
|
197
227
|
|
|
228
|
+
stem = enc_file.name.replace(".encoded.env", "").replace(".enc.env", "")
|
|
229
|
+
out_file = cwd / f"{stem}.env"
|
|
230
|
+
out_file.write_text(result.stdout)
|
|
198
231
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
"""Show help information."""
|
|
202
|
-
click.echo(__doc__)
|
|
232
|
+
click.echo(f" ✅ Decrypted -> {click.style(out_file.name, fg='cyan')}")
|
|
233
|
+
click.echo(f" 💡 Run: {click.style(f'source {out_file.name}', fg='yellow')}")
|
|
203
234
|
|
|
204
235
|
|
|
205
236
|
if __name__ == '__main__':
|
|
206
|
-
main()
|
|
237
|
+
main()
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import Protocol, List
|
|
3
|
+
from pathlib import PurePosixPath
|
|
4
|
+
import sys
|
|
5
|
+
import boto3
|
|
6
|
+
from botocore.client import Config
|
|
7
|
+
from botocore.exceptions import ClientError, EndpointConnectionError
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Storage(Protocol):
|
|
11
|
+
def put_file(self, local_path: str, path: str) -> None: ...
|
|
12
|
+
def get_file(self, path: str, local_path: str) -> None: ...
|
|
13
|
+
def put_bytes(self, data: bytes, path: str) -> None: ...
|
|
14
|
+
def get_bytes(self, path: str) -> bytes: ...
|
|
15
|
+
def list(self, prefix: str = "") -> List[str]: ...
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class S3Storage:
|
|
19
|
+
def __init__(self, tier: str = "bronze"):
|
|
20
|
+
self.tier = tier.lower()
|
|
21
|
+
|
|
22
|
+
if self.tier == "bronze":
|
|
23
|
+
access_key = os.getenv("BRONZE_ACCESS_KEY")
|
|
24
|
+
secret_key = os.getenv("BRONZE_SECRET_KEY")
|
|
25
|
+
self.bucket = os.getenv("BRONZE_BUCKET")
|
|
26
|
+
required_vars = ["BRONZE_ACCESS_KEY", "BRONZE_SECRET_KEY", "BRONZE_BUCKET"]
|
|
27
|
+
elif self.tier == "silver":
|
|
28
|
+
access_key = os.getenv("SILVER_ACCESS_KEY")
|
|
29
|
+
secret_key = os.getenv("SILVER_SECRET_KEY")
|
|
30
|
+
self.bucket = os.getenv("SILVER_BUCKET")
|
|
31
|
+
required_vars = ["SILVER_ACCESS_KEY", "SILVER_SECRET_KEY", "SILVER_BUCKET"]
|
|
32
|
+
else:
|
|
33
|
+
access_key = os.getenv("GOLD_ACCESS_KEY")
|
|
34
|
+
secret_key = os.getenv("GOLD_SECRET_KEY")
|
|
35
|
+
self.bucket = os.getenv("GOLD_BUCKET")
|
|
36
|
+
required_vars = ["GOLD_ACCESS_KEY", "GOLD_SECRET_KEY", "GOLD_BUCKET"]
|
|
37
|
+
|
|
38
|
+
missing_vars = [var for var in required_vars if not os.getenv(var)]
|
|
39
|
+
if missing_vars:
|
|
40
|
+
raise EnvironmentError(
|
|
41
|
+
f"Missing required environment variables for {self.tier} tier: {', '.join(missing_vars)}"
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
endpoint = os.getenv("S3_ENDPOINT", "http://localhost:9000")
|
|
45
|
+
|
|
46
|
+
self.client = boto3.client(
|
|
47
|
+
"s3",
|
|
48
|
+
endpoint_url=endpoint,
|
|
49
|
+
aws_access_key_id=access_key,
|
|
50
|
+
aws_secret_access_key=secret_key,
|
|
51
|
+
config=Config(signature_version="s3v4"),
|
|
52
|
+
region_name="us-east-1",
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
self.endpoint = endpoint
|
|
56
|
+
|
|
57
|
+
def _key(self, path: str) -> str:
|
|
58
|
+
return str(PurePosixPath(path).as_posix().lstrip("/"))
|
|
59
|
+
|
|
60
|
+
def _get_absolute_url(self, path: str) -> str:
|
|
61
|
+
key = self._key(path)
|
|
62
|
+
return f"{self.endpoint}/{self.bucket}/{key}"
|
|
63
|
+
|
|
64
|
+
def put_file(self, local_path: str, path: str) -> None:
|
|
65
|
+
absolute_url = self._get_absolute_url(path)
|
|
66
|
+
try:
|
|
67
|
+
self.client.upload_file(local_path, self.bucket, self._key(path))
|
|
68
|
+
print(f"🔗 File uploaded to: {absolute_url}")
|
|
69
|
+
except EndpointConnectionError:
|
|
70
|
+
print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
|
|
71
|
+
sys.exit(1)
|
|
72
|
+
except ClientError as e:
|
|
73
|
+
error_code = e.response['Error']['Code']
|
|
74
|
+
if error_code == 'AccessDenied':
|
|
75
|
+
print(f"❌ Access denied when uploading to: {path}", file=sys.stderr)
|
|
76
|
+
elif error_code == 'NoSuchBucket':
|
|
77
|
+
print(f"❌ Bucket not found: {self.bucket}", file=sys.stderr)
|
|
78
|
+
else:
|
|
79
|
+
print(f"❌ Storage error when uploading to: {path}: {e}", file=sys.stderr)
|
|
80
|
+
sys.exit(1)
|
|
81
|
+
|
|
82
|
+
def get_file(self, path: str, local_path: str) -> None:
|
|
83
|
+
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
|
84
|
+
absolute_url = self._get_absolute_url(path)
|
|
85
|
+
try:
|
|
86
|
+
self.client.download_file(self.bucket, self._key(path), local_path)
|
|
87
|
+
print(f"📥 File downloaded from: {absolute_url}")
|
|
88
|
+
except EndpointConnectionError:
|
|
89
|
+
print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
|
|
90
|
+
sys.exit(1)
|
|
91
|
+
except ClientError as e:
|
|
92
|
+
error_code = e.response['Error']['Code']
|
|
93
|
+
if error_code == 'NoSuchKey':
|
|
94
|
+
print(f"❌ File not found: {path}", file=sys.stderr)
|
|
95
|
+
elif error_code == 'AccessDenied':
|
|
96
|
+
print(f"❌ Access denied when reading: {path}", file=sys.stderr)
|
|
97
|
+
else:
|
|
98
|
+
print(f"❌ Storage error when reading: {path}: {e}", file=sys.stderr)
|
|
99
|
+
sys.exit(1)
|
|
100
|
+
|
|
101
|
+
def put_bytes(self, data: bytes, path: str) -> None:
|
|
102
|
+
absolute_url = self._get_absolute_url(path)
|
|
103
|
+
try:
|
|
104
|
+
self.client.put_object(
|
|
105
|
+
Bucket=self.bucket,
|
|
106
|
+
Key=self._key(path),
|
|
107
|
+
Body=data
|
|
108
|
+
)
|
|
109
|
+
print(f"💾 Data stored at: {absolute_url}")
|
|
110
|
+
except EndpointConnectionError:
|
|
111
|
+
print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
|
|
112
|
+
sys.exit(1)
|
|
113
|
+
except ClientError as e:
|
|
114
|
+
error_code = e.response['Error']['Code']
|
|
115
|
+
if error_code == 'AccessDenied':
|
|
116
|
+
print(f"❌ Access denied when writing to: {path}", file=sys.stderr)
|
|
117
|
+
elif error_code == 'NoSuchBucket':
|
|
118
|
+
print(f"❌ Bucket not found: {self.bucket}", file=sys.stderr)
|
|
119
|
+
else:
|
|
120
|
+
print(f"❌ Storage error when writing to: {path}: {e}", file=sys.stderr)
|
|
121
|
+
sys.exit(1)
|
|
122
|
+
|
|
123
|
+
def get_bytes(self, path: str) -> bytes:
|
|
124
|
+
absolute_url = self._get_absolute_url(path)
|
|
125
|
+
try:
|
|
126
|
+
obj = self.client.get_object(
|
|
127
|
+
Bucket=self.bucket,
|
|
128
|
+
Key=self._key(path)
|
|
129
|
+
)
|
|
130
|
+
print(f"📥 Data retrieved from: {absolute_url}")
|
|
131
|
+
return obj["Body"].read()
|
|
132
|
+
except EndpointConnectionError:
|
|
133
|
+
print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
|
|
134
|
+
sys.exit(1)
|
|
135
|
+
except ClientError as e:
|
|
136
|
+
error_code = e.response['Error']['Code']
|
|
137
|
+
if error_code == 'NoSuchKey':
|
|
138
|
+
print(f"❌ File not found: {path}", file=sys.stderr)
|
|
139
|
+
elif error_code == 'AccessDenied':
|
|
140
|
+
print(f"❌ Access denied when reading: {path}", file=sys.stderr)
|
|
141
|
+
else:
|
|
142
|
+
print(f"❌ Storage error when reading: {path}: {e}", file=sys.stderr)
|
|
143
|
+
sys.exit(1)
|
|
144
|
+
|
|
145
|
+
def list(self, prefix: str = "") -> List[str]:
|
|
146
|
+
resp = self.client.list_objects_v2(
|
|
147
|
+
Bucket=self.bucket,
|
|
148
|
+
Prefix=self._key(prefix)
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
return [
|
|
152
|
+
obj["Key"]
|
|
153
|
+
for obj in resp.get("Contents", [])
|
|
154
|
+
]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def get_storage(tier: str = "bronze") -> Storage:
|
|
158
|
+
return S3Storage(tier=tier)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steve-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
|
|
5
5
|
Author: Frank
|
|
6
6
|
License: MIT
|
|
@@ -17,12 +17,20 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
18
|
Requires-Python: >=3.8
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
|
+
Requires-Dist: boto3>=1.37.38
|
|
20
21
|
Requires-Dist: click>=8.0.0
|
|
21
22
|
Requires-Dist: pyyaml>=6.0
|
|
23
|
+
Requires-Dist: questionary>=2.0.0
|
|
22
24
|
Provides-Extra: dev
|
|
23
25
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
24
26
|
Requires-Dist: black>=22.0; extra == "dev"
|
|
25
27
|
Requires-Dist: isort>=5.0; extra == "dev"
|
|
28
|
+
Provides-Extra: polars
|
|
29
|
+
Requires-Dist: polars>=1.8.2; extra == "polars"
|
|
30
|
+
Requires-Dist: pyarrow>=17.0.0; extra == "polars"
|
|
31
|
+
Provides-Extra: pandas
|
|
32
|
+
Requires-Dist: pandas>=2.0.3; extra == "pandas"
|
|
33
|
+
Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
|
|
26
34
|
|
|
27
35
|
# Steve CLI
|
|
28
36
|
|
|
@@ -40,21 +48,28 @@ uv pip install -e .
|
|
|
40
48
|
|
|
41
49
|
### Install from GitHub
|
|
42
50
|
|
|
51
|
+
`uv add steve-cli`
|
|
52
|
+
|
|
53
|
+
OR
|
|
54
|
+
|
|
43
55
|
```bash
|
|
44
56
|
uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
|
|
45
57
|
```
|
|
46
58
|
|
|
59
|
+
|
|
60
|
+
|
|
47
61
|
## Usage
|
|
48
62
|
|
|
49
63
|
### Run a job
|
|
50
64
|
|
|
51
65
|
```bash
|
|
52
66
|
steve extract-data
|
|
53
|
-
steve transform-data
|
|
67
|
+
steve transform-data
|
|
54
68
|
steve validate-data
|
|
55
69
|
```
|
|
56
70
|
|
|
57
71
|
This will:
|
|
72
|
+
|
|
58
73
|
1. Look for `jobs.yaml` in the current directory
|
|
59
74
|
2. Find the specified job by name
|
|
60
75
|
3. Set all environment variables from the job's `env` section
|
|
@@ -94,7 +109,7 @@ jobs:
|
|
|
94
109
|
env:
|
|
95
110
|
SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
|
|
96
111
|
OUTPUT_PATH: "/data/raw/extracted_data.parquet"
|
|
97
|
-
|
|
112
|
+
|
|
98
113
|
- name: transform-data
|
|
99
114
|
dependsOn: ["extract-data"]
|
|
100
115
|
command: ["uv", "run", "src/transform.py"]
|
|
@@ -104,6 +119,7 @@ jobs:
|
|
|
104
119
|
```
|
|
105
120
|
|
|
106
121
|
Running `steve extract-data` will:
|
|
122
|
+
|
|
107
123
|
1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
|
|
108
124
|
2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
|
|
109
125
|
3. Execute: `uv run src/extract.py`
|
|
@@ -141,3 +157,14 @@ pytest
|
|
|
141
157
|
black steve_cli/
|
|
142
158
|
isort steve_cli/
|
|
143
159
|
```
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
# use cli
|
|
164
|
+
source .venv/bin/activate
|
|
165
|
+
steve
|
|
166
|
+
|
|
167
|
+
#
|
|
168
|
+
|
|
169
|
+
uv build
|
|
170
|
+
uv publish
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|