steve-cli 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {steve_cli-0.1.0 → steve_cli-0.2.0}/PKG-INFO +23 -3
- {steve_cli-0.1.0 → steve_cli-0.2.0}/README.md +16 -3
- {steve_cli-0.1.0 → steve_cli-0.2.0}/pyproject.toml +15 -2
- {steve_cli-0.1.0 → steve_cli-0.2.0}/steve_cli/cli.py +111 -100
- steve_cli-0.2.0/steve_cli/storage.py +158 -0
- {steve_cli-0.1.0 → steve_cli-0.2.0}/steve_cli.egg-info/PKG-INFO +23 -3
- {steve_cli-0.1.0 → steve_cli-0.2.0}/steve_cli.egg-info/SOURCES.txt +1 -0
- steve_cli-0.2.0/steve_cli.egg-info/requires.txt +16 -0
- steve_cli-0.1.0/steve_cli.egg-info/requires.txt +0 -7
- {steve_cli-0.1.0 → steve_cli-0.2.0}/setup.cfg +0 -0
- {steve_cli-0.1.0 → steve_cli-0.2.0}/steve_cli/__init__.py +0 -0
- {steve_cli-0.1.0 → steve_cli-0.2.0}/steve_cli.egg-info/dependency_links.txt +0 -0
- {steve_cli-0.1.0 → steve_cli-0.2.0}/steve_cli.egg-info/entry_points.txt +0 -0
- {steve_cli-0.1.0 → steve_cli-0.2.0}/steve_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steve-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
|
|
5
5
|
Author: Frank
|
|
6
6
|
License: MIT
|
|
@@ -17,12 +17,19 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
18
|
Requires-Python: >=3.8
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
|
+
Requires-Dist: boto3>=1.37.38
|
|
20
21
|
Requires-Dist: click>=8.0.0
|
|
21
22
|
Requires-Dist: pyyaml>=6.0
|
|
22
23
|
Provides-Extra: dev
|
|
23
24
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
24
25
|
Requires-Dist: black>=22.0; extra == "dev"
|
|
25
26
|
Requires-Dist: isort>=5.0; extra == "dev"
|
|
27
|
+
Provides-Extra: polars
|
|
28
|
+
Requires-Dist: polars>=1.8.2; extra == "polars"
|
|
29
|
+
Requires-Dist: pyarrow>=17.0.0; extra == "polars"
|
|
30
|
+
Provides-Extra: pandas
|
|
31
|
+
Requires-Dist: pandas>=2.0.3; extra == "pandas"
|
|
32
|
+
Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
|
|
26
33
|
|
|
27
34
|
# Steve CLI
|
|
28
35
|
|
|
@@ -40,21 +47,28 @@ uv pip install -e .
|
|
|
40
47
|
|
|
41
48
|
### Install from GitHub
|
|
42
49
|
|
|
50
|
+
`uv add steve-cli`
|
|
51
|
+
|
|
52
|
+
OR
|
|
53
|
+
|
|
43
54
|
```bash
|
|
44
55
|
uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
|
|
45
56
|
```
|
|
46
57
|
|
|
58
|
+
|
|
59
|
+
|
|
47
60
|
## Usage
|
|
48
61
|
|
|
49
62
|
### Run a job
|
|
50
63
|
|
|
51
64
|
```bash
|
|
52
65
|
steve extract-data
|
|
53
|
-
steve transform-data
|
|
66
|
+
steve transform-data
|
|
54
67
|
steve validate-data
|
|
55
68
|
```
|
|
56
69
|
|
|
57
70
|
This will:
|
|
71
|
+
|
|
58
72
|
1. Look for `jobs.yaml` in the current directory
|
|
59
73
|
2. Find the specified job by name
|
|
60
74
|
3. Set all environment variables from the job's `env` section
|
|
@@ -94,7 +108,7 @@ jobs:
|
|
|
94
108
|
env:
|
|
95
109
|
SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
|
|
96
110
|
OUTPUT_PATH: "/data/raw/extracted_data.parquet"
|
|
97
|
-
|
|
111
|
+
|
|
98
112
|
- name: transform-data
|
|
99
113
|
dependsOn: ["extract-data"]
|
|
100
114
|
command: ["uv", "run", "src/transform.py"]
|
|
@@ -104,6 +118,7 @@ jobs:
|
|
|
104
118
|
```
|
|
105
119
|
|
|
106
120
|
Running `steve extract-data` will:
|
|
121
|
+
|
|
107
122
|
1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
|
|
108
123
|
2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
|
|
109
124
|
3. Execute: `uv run src/extract.py`
|
|
@@ -141,3 +156,8 @@ pytest
|
|
|
141
156
|
black steve_cli/
|
|
142
157
|
isort steve_cli/
|
|
143
158
|
```
|
|
159
|
+
|
|
160
|
+
#
|
|
161
|
+
|
|
162
|
+
uv build
|
|
163
|
+
uv publish
|
|
@@ -14,21 +14,28 @@ uv pip install -e .
|
|
|
14
14
|
|
|
15
15
|
### Install from GitHub
|
|
16
16
|
|
|
17
|
+
`uv add steve-cli`
|
|
18
|
+
|
|
19
|
+
OR
|
|
20
|
+
|
|
17
21
|
```bash
|
|
18
22
|
uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
|
|
19
23
|
```
|
|
20
24
|
|
|
25
|
+
|
|
26
|
+
|
|
21
27
|
## Usage
|
|
22
28
|
|
|
23
29
|
### Run a job
|
|
24
30
|
|
|
25
31
|
```bash
|
|
26
32
|
steve extract-data
|
|
27
|
-
steve transform-data
|
|
33
|
+
steve transform-data
|
|
28
34
|
steve validate-data
|
|
29
35
|
```
|
|
30
36
|
|
|
31
37
|
This will:
|
|
38
|
+
|
|
32
39
|
1. Look for `jobs.yaml` in the current directory
|
|
33
40
|
2. Find the specified job by name
|
|
34
41
|
3. Set all environment variables from the job's `env` section
|
|
@@ -68,7 +75,7 @@ jobs:
|
|
|
68
75
|
env:
|
|
69
76
|
SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
|
|
70
77
|
OUTPUT_PATH: "/data/raw/extracted_data.parquet"
|
|
71
|
-
|
|
78
|
+
|
|
72
79
|
- name: transform-data
|
|
73
80
|
dependsOn: ["extract-data"]
|
|
74
81
|
command: ["uv", "run", "src/transform.py"]
|
|
@@ -78,6 +85,7 @@ jobs:
|
|
|
78
85
|
```
|
|
79
86
|
|
|
80
87
|
Running `steve extract-data` will:
|
|
88
|
+
|
|
81
89
|
1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
|
|
82
90
|
2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
|
|
83
91
|
3. Execute: `uv run src/extract.py`
|
|
@@ -114,4 +122,9 @@ pytest
|
|
|
114
122
|
# Format code
|
|
115
123
|
black steve_cli/
|
|
116
124
|
isort steve_cli/
|
|
117
|
-
```
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
#
|
|
128
|
+
|
|
129
|
+
uv build
|
|
130
|
+
uv publish
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
[project]
|
|
3
3
|
name = "steve-cli"
|
|
4
|
-
version = "0.
|
|
4
|
+
version = "0.2.0"
|
|
5
5
|
description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
|
|
6
6
|
readme = "README.md"
|
|
7
7
|
license = {text = "MIT"}
|
|
@@ -19,6 +19,7 @@ classifiers = [
|
|
|
19
19
|
]
|
|
20
20
|
requires-python = ">=3.8"
|
|
21
21
|
dependencies = [
|
|
22
|
+
"boto3>=1.37.38",
|
|
22
23
|
"click>=8.0.0",
|
|
23
24
|
"pyyaml>=6.0",
|
|
24
25
|
]
|
|
@@ -29,6 +30,14 @@ dev = [
|
|
|
29
30
|
"black>=22.0",
|
|
30
31
|
"isort>=5.0",
|
|
31
32
|
]
|
|
33
|
+
polars = [
|
|
34
|
+
"polars>=1.8.2",
|
|
35
|
+
"pyarrow>=17.0.0",
|
|
36
|
+
]
|
|
37
|
+
pandas = [
|
|
38
|
+
"pandas>=2.0.3",
|
|
39
|
+
"pyarrow>=17.0.0",
|
|
40
|
+
]
|
|
32
41
|
|
|
33
42
|
[project.scripts]
|
|
34
43
|
steve = "steve_cli.cli:main"
|
|
@@ -39,9 +48,13 @@ Repository = "https://github.com/7frank/ds-steve-cli"
|
|
|
39
48
|
|
|
40
49
|
|
|
41
50
|
|
|
51
|
+
[tool.setuptools.packages.find]
|
|
52
|
+
where = ["."]
|
|
53
|
+
include = ["steve_cli*"]
|
|
54
|
+
|
|
42
55
|
[tool.black]
|
|
43
56
|
line-length = 88
|
|
44
57
|
target-version = ['py38']
|
|
45
58
|
|
|
46
59
|
[tool.isort]
|
|
47
|
-
profile = "black"
|
|
60
|
+
profile = "black"
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
Steve CLI - A simple tool to run jobs from jobs.yaml with proper environment setup.
|
|
4
4
|
|
|
5
5
|
Usage:
|
|
6
|
-
steve
|
|
7
|
-
steve
|
|
8
|
-
steve
|
|
6
|
+
steve jobs ls List all available jobs
|
|
7
|
+
steve jobs run <job-name> Run a job with its environment variables
|
|
8
|
+
steve setup all Decrypt SOPS-encoded .env files
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import os
|
|
@@ -19,86 +19,69 @@ import yaml
|
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
class JobsConfig:
|
|
22
|
-
"""Handle parsing and querying of jobs.yaml configuration."""
|
|
23
|
-
|
|
24
22
|
def __init__(self, jobs_file: Optional[Path] = None):
|
|
25
23
|
self.jobs_file = jobs_file or Path.cwd() / "jobs.yaml"
|
|
26
24
|
self.jobs: List[Dict[str, Any]] = []
|
|
27
25
|
self._load_jobs()
|
|
28
|
-
|
|
26
|
+
|
|
29
27
|
def _load_jobs(self) -> None:
|
|
30
|
-
"""Load and parse the jobs.yaml file."""
|
|
31
28
|
try:
|
|
32
29
|
if not self.jobs_file.exists():
|
|
33
30
|
click.echo(f"❌ Error: jobs.yaml not found at {self.jobs_file}", err=True)
|
|
34
31
|
click.echo("Make sure you're in a directory with a jobs.yaml file", err=True)
|
|
35
32
|
sys.exit(1)
|
|
36
|
-
|
|
33
|
+
|
|
37
34
|
with open(self.jobs_file, 'r') as f:
|
|
38
35
|
data = yaml.safe_load(f)
|
|
39
|
-
|
|
36
|
+
|
|
40
37
|
if not data or 'jobs' not in data:
|
|
41
|
-
click.echo(
|
|
38
|
+
click.echo("❌ Error: Invalid jobs.yaml format. Expected 'jobs' key at root", err=True)
|
|
42
39
|
sys.exit(1)
|
|
43
|
-
|
|
40
|
+
|
|
44
41
|
self.jobs = data['jobs']
|
|
45
|
-
|
|
42
|
+
|
|
46
43
|
except yaml.YAMLError as e:
|
|
47
44
|
click.echo(f"❌ Error parsing jobs.yaml: {e}", err=True)
|
|
48
45
|
sys.exit(1)
|
|
49
46
|
except Exception as e:
|
|
50
47
|
click.echo(f"❌ Error reading jobs.yaml: {e}", err=True)
|
|
51
48
|
sys.exit(1)
|
|
52
|
-
|
|
49
|
+
|
|
53
50
|
def get_job(self, job_name: str) -> Optional[Dict[str, Any]]:
|
|
54
|
-
"""Get a job by name."""
|
|
55
51
|
for job in self.jobs:
|
|
56
52
|
if job.get('name') == job_name:
|
|
57
53
|
return job
|
|
58
54
|
return None
|
|
59
|
-
|
|
55
|
+
|
|
60
56
|
def list_jobs(self) -> List[str]:
|
|
61
|
-
"""Get list of all job names."""
|
|
62
57
|
return [job.get('name', 'unnamed') for job in self.jobs]
|
|
63
58
|
|
|
64
59
|
|
|
65
60
|
def run_job_command(job: Dict[str, Any]) -> int:
|
|
66
|
-
"""Run a job's command with its environment variables."""
|
|
67
|
-
# Get command
|
|
68
61
|
command = job.get('command', [])
|
|
69
62
|
if not command:
|
|
70
63
|
click.echo(f"❌ Error: No command specified for job '{job.get('name')}'", err=True)
|
|
71
64
|
return 1
|
|
72
|
-
|
|
73
|
-
# Prepare environment
|
|
65
|
+
|
|
74
66
|
env = os.environ.copy()
|
|
75
67
|
job_env = job.get('env', {})
|
|
76
|
-
|
|
77
|
-
# Convert all env values to strings and merge
|
|
78
68
|
for key, value in job_env.items():
|
|
79
69
|
env[key] = str(value)
|
|
80
|
-
|
|
81
|
-
# Show what we're running
|
|
70
|
+
|
|
82
71
|
job_name = job.get('name', 'unnamed')
|
|
83
72
|
click.echo(f"🚀 Running job: {click.style(job_name, fg='blue', bold=True)}")
|
|
84
73
|
click.echo(f"📝 Command: {click.style(' '.join(command), fg='cyan')}")
|
|
85
|
-
|
|
74
|
+
|
|
86
75
|
if job_env:
|
|
87
|
-
click.echo(
|
|
76
|
+
click.echo("🌍 Environment variables:")
|
|
88
77
|
for key, value in job_env.items():
|
|
89
78
|
click.echo(f" {click.style(key, fg='green')}={click.style(str(value), fg='yellow')}")
|
|
90
|
-
|
|
91
|
-
click.echo()
|
|
92
|
-
|
|
79
|
+
|
|
80
|
+
click.echo()
|
|
81
|
+
|
|
93
82
|
try:
|
|
94
|
-
|
|
95
|
-
result = subprocess.run(
|
|
96
|
-
command,
|
|
97
|
-
env=env,
|
|
98
|
-
cwd=Path.cwd(),
|
|
99
|
-
)
|
|
83
|
+
result = subprocess.run(command, env=env, cwd=Path.cwd())
|
|
100
84
|
return result.returncode
|
|
101
|
-
|
|
102
85
|
except FileNotFoundError:
|
|
103
86
|
click.echo(f"❌ Error: Command not found: {command[0]}", err=True)
|
|
104
87
|
return 127
|
|
@@ -107,100 +90,128 @@ def run_job_command(job: Dict[str, Any]) -> int:
|
|
|
107
90
|
return 1
|
|
108
91
|
|
|
109
92
|
|
|
110
|
-
@click.group(
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def main(ctx: click.Context, jobs_file: Optional[Path], job_name: Optional[str]):
|
|
116
|
-
"""Steve CLI - Run jobs from jobs.yaml with proper environment setup."""
|
|
117
|
-
|
|
118
|
-
# Handle the case where a job name is provided directly
|
|
119
|
-
if job_name:
|
|
120
|
-
config = JobsConfig(jobs_file)
|
|
121
|
-
|
|
122
|
-
# Special cases for built-in commands
|
|
123
|
-
if job_name in ['ls', 'list']:
|
|
124
|
-
list_jobs_command(config)
|
|
125
|
-
return
|
|
126
|
-
elif job_name in ['help', '--help', '-h']:
|
|
127
|
-
click.echo(ctx.get_help())
|
|
128
|
-
return
|
|
129
|
-
|
|
130
|
-
# Try to run the job
|
|
131
|
-
job = config.get_job(job_name)
|
|
132
|
-
if not job:
|
|
133
|
-
available_jobs = config.list_jobs()
|
|
134
|
-
click.echo(f"❌ Error: Job '{job_name}' not found", err=True)
|
|
135
|
-
if available_jobs:
|
|
136
|
-
click.echo(f"Available jobs: {', '.join(available_jobs)}", err=True)
|
|
137
|
-
else:
|
|
138
|
-
click.echo("No jobs found in jobs.yaml", err=True)
|
|
139
|
-
sys.exit(1)
|
|
140
|
-
|
|
141
|
-
exit_code = run_job_command(job)
|
|
142
|
-
sys.exit(exit_code)
|
|
143
|
-
|
|
144
|
-
# If no job name provided, show help
|
|
145
|
-
click.echo(ctx.get_help())
|
|
93
|
+
@click.group()
|
|
94
|
+
def main():
|
|
95
|
+
"""Steve CLI - Run jobs and manage environment setup."""
|
|
96
|
+
pass
|
|
97
|
+
|
|
146
98
|
|
|
99
|
+
@main.group()
|
|
100
|
+
def jobs():
|
|
101
|
+
"""Manage and run jobs from jobs.yaml."""
|
|
102
|
+
pass
|
|
147
103
|
|
|
148
|
-
|
|
104
|
+
|
|
105
|
+
@jobs.command('ls')
|
|
149
106
|
@click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
|
|
150
107
|
help='Path to jobs.yaml file (default: ./jobs.yaml)')
|
|
151
|
-
def
|
|
108
|
+
def jobs_list(jobs_file: Optional[Path]):
|
|
152
109
|
"""List all available jobs."""
|
|
153
110
|
config = JobsConfig(jobs_file)
|
|
154
|
-
|
|
155
|
-
|
|
111
|
+
job_names = config.list_jobs()
|
|
156
112
|
|
|
157
|
-
|
|
158
|
-
"""Show list of available jobs with details."""
|
|
159
|
-
jobs = config.list_jobs()
|
|
160
|
-
|
|
161
|
-
if not jobs:
|
|
113
|
+
if not job_names:
|
|
162
114
|
click.echo("❌ No jobs found in jobs.yaml")
|
|
163
115
|
return
|
|
164
|
-
|
|
116
|
+
|
|
165
117
|
click.echo(f"📋 Available jobs in {click.style(str(config.jobs_file), fg='cyan')}:")
|
|
166
118
|
click.echo()
|
|
167
|
-
|
|
119
|
+
|
|
168
120
|
for job_data in config.jobs:
|
|
169
121
|
name = job_data.get('name', 'unnamed')
|
|
170
122
|
command = job_data.get('command', [])
|
|
171
123
|
cron = job_data.get('cron')
|
|
172
124
|
depends_on = job_data.get('dependsOn', [])
|
|
173
125
|
env_vars = job_data.get('env', {})
|
|
174
|
-
|
|
175
|
-
# Job name
|
|
126
|
+
|
|
176
127
|
click.echo(f" {click.style(name, fg='blue', bold=True)}")
|
|
177
|
-
|
|
178
|
-
# Command
|
|
179
128
|
if command:
|
|
180
129
|
click.echo(f" Command: {click.style(' '.join(command), fg='cyan')}")
|
|
181
|
-
|
|
182
|
-
# Schedule
|
|
183
130
|
if cron:
|
|
184
131
|
click.echo(f" Schedule: {click.style(cron, fg='yellow')}")
|
|
185
|
-
|
|
186
|
-
# Dependencies
|
|
187
132
|
if depends_on:
|
|
188
133
|
click.echo(f" Depends on: {click.style(', '.join(depends_on), fg='magenta')}")
|
|
189
|
-
|
|
190
|
-
# Environment variables
|
|
191
134
|
if env_vars:
|
|
192
135
|
click.echo(f" Environment: {click.style(f'{len(env_vars)} variables', fg='green')}")
|
|
193
136
|
for key, value in env_vars.items():
|
|
194
137
|
click.echo(f" {key}={value}")
|
|
195
|
-
|
|
196
|
-
|
|
138
|
+
click.echo()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@jobs.command('run')
|
|
142
|
+
@click.argument('job_name')
|
|
143
|
+
@click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
|
|
144
|
+
help='Path to jobs.yaml file (default: ./jobs.yaml)')
|
|
145
|
+
def jobs_run(job_name: str, jobs_file: Optional[Path]):
|
|
146
|
+
"""Run a job by name."""
|
|
147
|
+
config = JobsConfig(jobs_file)
|
|
148
|
+
job = config.get_job(job_name)
|
|
149
|
+
if not job:
|
|
150
|
+
available = config.list_jobs()
|
|
151
|
+
click.echo(f"❌ Error: Job '{job_name}' not found", err=True)
|
|
152
|
+
if available:
|
|
153
|
+
click.echo(f"Available jobs: {', '.join(available)}", err=True)
|
|
154
|
+
sys.exit(1)
|
|
155
|
+
sys.exit(run_job_command(job))
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
SETUP_IGNORE = [".env.example", ".env.template"]
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@main.group()
|
|
162
|
+
def setup():
|
|
163
|
+
"""Setup commands for environment configuration."""
|
|
164
|
+
pass
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@setup.command("all")
|
|
168
|
+
def setup_all():
|
|
169
|
+
"""Decrypt SOPS-encoded .env files in current directory and write plaintext .env files."""
|
|
170
|
+
cwd = Path.cwd()
|
|
171
|
+
found: List[Path] = []
|
|
172
|
+
for pattern in ["*.enc.env", "*.encoded.env"]:
|
|
173
|
+
found.extend(sorted(cwd.glob(pattern)))
|
|
174
|
+
|
|
175
|
+
found = [f for f in found if f.name not in SETUP_IGNORE]
|
|
176
|
+
|
|
177
|
+
if not found:
|
|
178
|
+
click.echo("No *.enc.env or *.encoded.env files found.")
|
|
179
|
+
return
|
|
180
|
+
|
|
181
|
+
for enc_file in found:
|
|
182
|
+
click.echo(f"\n📄 Found: {click.style(enc_file.name, fg='blue', bold=True)}")
|
|
183
|
+
|
|
184
|
+
content = enc_file.read_text()
|
|
185
|
+
if "ENC[" not in content and "sops:" not in content:
|
|
186
|
+
click.echo(" ⚠️ Not SOPS encrypted, skipping.")
|
|
187
|
+
continue
|
|
188
|
+
|
|
189
|
+
result = subprocess.run(
|
|
190
|
+
["sops", "-d", str(enc_file)],
|
|
191
|
+
capture_output=True,
|
|
192
|
+
text=True,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
if result.returncode != 0:
|
|
196
|
+
click.echo(f" ❌ Decryption failed: {result.stderr.strip()}", err=True)
|
|
197
|
+
continue
|
|
198
|
+
|
|
199
|
+
keys = []
|
|
200
|
+
for line in result.stdout.splitlines():
|
|
201
|
+
line = line.strip()
|
|
202
|
+
if line and not line.startswith("#") and "=" in line:
|
|
203
|
+
keys.append(line.split("=", 1)[0])
|
|
204
|
+
|
|
205
|
+
if keys:
|
|
206
|
+
click.echo(f" 🔑 Keys: {click.style(', '.join(keys), fg='green')}")
|
|
197
207
|
|
|
208
|
+
stem = enc_file.name.replace(".encoded.env", "").replace(".enc.env", "")
|
|
209
|
+
out_file = cwd / f"{stem}.env"
|
|
210
|
+
out_file.write_text(result.stdout)
|
|
198
211
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
"""Show help information."""
|
|
202
|
-
click.echo(__doc__)
|
|
212
|
+
click.echo(f" ✅ Decrypted -> {click.style(out_file.name, fg='cyan')}")
|
|
213
|
+
click.echo(f" 💡 Run: {click.style(f'source {out_file.name}', fg='yellow')}")
|
|
203
214
|
|
|
204
215
|
|
|
205
216
|
if __name__ == '__main__':
|
|
206
|
-
main()
|
|
217
|
+
main()
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import Protocol, List
|
|
3
|
+
from pathlib import PurePosixPath
|
|
4
|
+
import sys
|
|
5
|
+
import boto3
|
|
6
|
+
from botocore.client import Config
|
|
7
|
+
from botocore.exceptions import ClientError, EndpointConnectionError
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Storage(Protocol):
|
|
11
|
+
def put_file(self, local_path: str, path: str) -> None: ...
|
|
12
|
+
def get_file(self, path: str, local_path: str) -> None: ...
|
|
13
|
+
def put_bytes(self, data: bytes, path: str) -> None: ...
|
|
14
|
+
def get_bytes(self, path: str) -> bytes: ...
|
|
15
|
+
def list(self, prefix: str = "") -> List[str]: ...
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class S3Storage:
|
|
19
|
+
def __init__(self, tier: str = "bronze"):
|
|
20
|
+
self.tier = tier.lower()
|
|
21
|
+
|
|
22
|
+
if self.tier == "bronze":
|
|
23
|
+
access_key = os.getenv("BRONZE_ACCESS_KEY")
|
|
24
|
+
secret_key = os.getenv("BRONZE_SECRET_KEY")
|
|
25
|
+
self.bucket = os.getenv("BRONZE_BUCKET")
|
|
26
|
+
required_vars = ["BRONZE_ACCESS_KEY", "BRONZE_SECRET_KEY", "BRONZE_BUCKET"]
|
|
27
|
+
elif self.tier == "silver":
|
|
28
|
+
access_key = os.getenv("SILVER_ACCESS_KEY")
|
|
29
|
+
secret_key = os.getenv("SILVER_SECRET_KEY")
|
|
30
|
+
self.bucket = os.getenv("SILVER_BUCKET")
|
|
31
|
+
required_vars = ["SILVER_ACCESS_KEY", "SILVER_SECRET_KEY", "SILVER_BUCKET"]
|
|
32
|
+
else:
|
|
33
|
+
access_key = os.getenv("GOLD_ACCESS_KEY")
|
|
34
|
+
secret_key = os.getenv("GOLD_SECRET_KEY")
|
|
35
|
+
self.bucket = os.getenv("GOLD_BUCKET")
|
|
36
|
+
required_vars = ["GOLD_ACCESS_KEY", "GOLD_SECRET_KEY", "GOLD_BUCKET"]
|
|
37
|
+
|
|
38
|
+
missing_vars = [var for var in required_vars if not os.getenv(var)]
|
|
39
|
+
if missing_vars:
|
|
40
|
+
raise EnvironmentError(
|
|
41
|
+
f"Missing required environment variables for {self.tier} tier: {', '.join(missing_vars)}"
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
endpoint = os.getenv("S3_ENDPOINT", "http://localhost:9000")
|
|
45
|
+
|
|
46
|
+
self.client = boto3.client(
|
|
47
|
+
"s3",
|
|
48
|
+
endpoint_url=endpoint,
|
|
49
|
+
aws_access_key_id=access_key,
|
|
50
|
+
aws_secret_access_key=secret_key,
|
|
51
|
+
config=Config(signature_version="s3v4"),
|
|
52
|
+
region_name="us-east-1",
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
self.endpoint = endpoint
|
|
56
|
+
|
|
57
|
+
def _key(self, path: str) -> str:
|
|
58
|
+
return str(PurePosixPath(path).as_posix().lstrip("/"))
|
|
59
|
+
|
|
60
|
+
def _get_absolute_url(self, path: str) -> str:
|
|
61
|
+
key = self._key(path)
|
|
62
|
+
return f"{self.endpoint}/{self.bucket}/{key}"
|
|
63
|
+
|
|
64
|
+
def put_file(self, local_path: str, path: str) -> None:
|
|
65
|
+
absolute_url = self._get_absolute_url(path)
|
|
66
|
+
try:
|
|
67
|
+
self.client.upload_file(local_path, self.bucket, self._key(path))
|
|
68
|
+
print(f"🔗 File uploaded to: {absolute_url}")
|
|
69
|
+
except EndpointConnectionError:
|
|
70
|
+
print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
|
|
71
|
+
sys.exit(1)
|
|
72
|
+
except ClientError as e:
|
|
73
|
+
error_code = e.response['Error']['Code']
|
|
74
|
+
if error_code == 'AccessDenied':
|
|
75
|
+
print(f"❌ Access denied when uploading to: {path}", file=sys.stderr)
|
|
76
|
+
elif error_code == 'NoSuchBucket':
|
|
77
|
+
print(f"❌ Bucket not found: {self.bucket}", file=sys.stderr)
|
|
78
|
+
else:
|
|
79
|
+
print(f"❌ Storage error when uploading to: {path}: {e}", file=sys.stderr)
|
|
80
|
+
sys.exit(1)
|
|
81
|
+
|
|
82
|
+
def get_file(self, path: str, local_path: str) -> None:
|
|
83
|
+
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
|
84
|
+
absolute_url = self._get_absolute_url(path)
|
|
85
|
+
try:
|
|
86
|
+
self.client.download_file(self.bucket, self._key(path), local_path)
|
|
87
|
+
print(f"📥 File downloaded from: {absolute_url}")
|
|
88
|
+
except EndpointConnectionError:
|
|
89
|
+
print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
|
|
90
|
+
sys.exit(1)
|
|
91
|
+
except ClientError as e:
|
|
92
|
+
error_code = e.response['Error']['Code']
|
|
93
|
+
if error_code == 'NoSuchKey':
|
|
94
|
+
print(f"❌ File not found: {path}", file=sys.stderr)
|
|
95
|
+
elif error_code == 'AccessDenied':
|
|
96
|
+
print(f"❌ Access denied when reading: {path}", file=sys.stderr)
|
|
97
|
+
else:
|
|
98
|
+
print(f"❌ Storage error when reading: {path}: {e}", file=sys.stderr)
|
|
99
|
+
sys.exit(1)
|
|
100
|
+
|
|
101
|
+
def put_bytes(self, data: bytes, path: str) -> None:
|
|
102
|
+
absolute_url = self._get_absolute_url(path)
|
|
103
|
+
try:
|
|
104
|
+
self.client.put_object(
|
|
105
|
+
Bucket=self.bucket,
|
|
106
|
+
Key=self._key(path),
|
|
107
|
+
Body=data
|
|
108
|
+
)
|
|
109
|
+
print(f"💾 Data stored at: {absolute_url}")
|
|
110
|
+
except EndpointConnectionError:
|
|
111
|
+
print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
|
|
112
|
+
sys.exit(1)
|
|
113
|
+
except ClientError as e:
|
|
114
|
+
error_code = e.response['Error']['Code']
|
|
115
|
+
if error_code == 'AccessDenied':
|
|
116
|
+
print(f"❌ Access denied when writing to: {path}", file=sys.stderr)
|
|
117
|
+
elif error_code == 'NoSuchBucket':
|
|
118
|
+
print(f"❌ Bucket not found: {self.bucket}", file=sys.stderr)
|
|
119
|
+
else:
|
|
120
|
+
print(f"❌ Storage error when writing to: {path}: {e}", file=sys.stderr)
|
|
121
|
+
sys.exit(1)
|
|
122
|
+
|
|
123
|
+
def get_bytes(self, path: str) -> bytes:
|
|
124
|
+
absolute_url = self._get_absolute_url(path)
|
|
125
|
+
try:
|
|
126
|
+
obj = self.client.get_object(
|
|
127
|
+
Bucket=self.bucket,
|
|
128
|
+
Key=self._key(path)
|
|
129
|
+
)
|
|
130
|
+
print(f"📥 Data retrieved from: {absolute_url}")
|
|
131
|
+
return obj["Body"].read()
|
|
132
|
+
except EndpointConnectionError:
|
|
133
|
+
print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
|
|
134
|
+
sys.exit(1)
|
|
135
|
+
except ClientError as e:
|
|
136
|
+
error_code = e.response['Error']['Code']
|
|
137
|
+
if error_code == 'NoSuchKey':
|
|
138
|
+
print(f"❌ File not found: {path}", file=sys.stderr)
|
|
139
|
+
elif error_code == 'AccessDenied':
|
|
140
|
+
print(f"❌ Access denied when reading: {path}", file=sys.stderr)
|
|
141
|
+
else:
|
|
142
|
+
print(f"❌ Storage error when reading: {path}: {e}", file=sys.stderr)
|
|
143
|
+
sys.exit(1)
|
|
144
|
+
|
|
145
|
+
def list(self, prefix: str = "") -> List[str]:
|
|
146
|
+
resp = self.client.list_objects_v2(
|
|
147
|
+
Bucket=self.bucket,
|
|
148
|
+
Prefix=self._key(prefix)
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
return [
|
|
152
|
+
obj["Key"]
|
|
153
|
+
for obj in resp.get("Contents", [])
|
|
154
|
+
]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def get_storage(tier: str = "bronze") -> Storage:
|
|
158
|
+
return S3Storage(tier=tier)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steve-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
|
|
5
5
|
Author: Frank
|
|
6
6
|
License: MIT
|
|
@@ -17,12 +17,19 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
18
|
Requires-Python: >=3.8
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
|
+
Requires-Dist: boto3>=1.37.38
|
|
20
21
|
Requires-Dist: click>=8.0.0
|
|
21
22
|
Requires-Dist: pyyaml>=6.0
|
|
22
23
|
Provides-Extra: dev
|
|
23
24
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
24
25
|
Requires-Dist: black>=22.0; extra == "dev"
|
|
25
26
|
Requires-Dist: isort>=5.0; extra == "dev"
|
|
27
|
+
Provides-Extra: polars
|
|
28
|
+
Requires-Dist: polars>=1.8.2; extra == "polars"
|
|
29
|
+
Requires-Dist: pyarrow>=17.0.0; extra == "polars"
|
|
30
|
+
Provides-Extra: pandas
|
|
31
|
+
Requires-Dist: pandas>=2.0.3; extra == "pandas"
|
|
32
|
+
Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
|
|
26
33
|
|
|
27
34
|
# Steve CLI
|
|
28
35
|
|
|
@@ -40,21 +47,28 @@ uv pip install -e .
|
|
|
40
47
|
|
|
41
48
|
### Install from GitHub
|
|
42
49
|
|
|
50
|
+
`uv add steve-cli`
|
|
51
|
+
|
|
52
|
+
OR
|
|
53
|
+
|
|
43
54
|
```bash
|
|
44
55
|
uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
|
|
45
56
|
```
|
|
46
57
|
|
|
58
|
+
|
|
59
|
+
|
|
47
60
|
## Usage
|
|
48
61
|
|
|
49
62
|
### Run a job
|
|
50
63
|
|
|
51
64
|
```bash
|
|
52
65
|
steve extract-data
|
|
53
|
-
steve transform-data
|
|
66
|
+
steve transform-data
|
|
54
67
|
steve validate-data
|
|
55
68
|
```
|
|
56
69
|
|
|
57
70
|
This will:
|
|
71
|
+
|
|
58
72
|
1. Look for `jobs.yaml` in the current directory
|
|
59
73
|
2. Find the specified job by name
|
|
60
74
|
3. Set all environment variables from the job's `env` section
|
|
@@ -94,7 +108,7 @@ jobs:
|
|
|
94
108
|
env:
|
|
95
109
|
SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
|
|
96
110
|
OUTPUT_PATH: "/data/raw/extracted_data.parquet"
|
|
97
|
-
|
|
111
|
+
|
|
98
112
|
- name: transform-data
|
|
99
113
|
dependsOn: ["extract-data"]
|
|
100
114
|
command: ["uv", "run", "src/transform.py"]
|
|
@@ -104,6 +118,7 @@ jobs:
|
|
|
104
118
|
```
|
|
105
119
|
|
|
106
120
|
Running `steve extract-data` will:
|
|
121
|
+
|
|
107
122
|
1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
|
|
108
123
|
2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
|
|
109
124
|
3. Execute: `uv run src/extract.py`
|
|
@@ -141,3 +156,8 @@ pytest
|
|
|
141
156
|
black steve_cli/
|
|
142
157
|
isort steve_cli/
|
|
143
158
|
```
|
|
159
|
+
|
|
160
|
+
#
|
|
161
|
+
|
|
162
|
+
uv build
|
|
163
|
+
uv publish
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|