steve-cli 0.1.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 0.1.0
3
+ Version: 0.3.0
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -17,12 +17,20 @@ Classifier: Programming Language :: Python :: 3.11
17
17
  Classifier: Programming Language :: Python :: 3.12
18
18
  Requires-Python: >=3.8
19
19
  Description-Content-Type: text/markdown
20
+ Requires-Dist: boto3>=1.37.38
20
21
  Requires-Dist: click>=8.0.0
21
22
  Requires-Dist: pyyaml>=6.0
23
+ Requires-Dist: questionary>=2.0.0
22
24
  Provides-Extra: dev
23
25
  Requires-Dist: pytest>=7.0; extra == "dev"
24
26
  Requires-Dist: black>=22.0; extra == "dev"
25
27
  Requires-Dist: isort>=5.0; extra == "dev"
28
+ Provides-Extra: polars
29
+ Requires-Dist: polars>=1.8.2; extra == "polars"
30
+ Requires-Dist: pyarrow>=17.0.0; extra == "polars"
31
+ Provides-Extra: pandas
32
+ Requires-Dist: pandas>=2.0.3; extra == "pandas"
33
+ Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
26
34
 
27
35
  # Steve CLI
28
36
 
@@ -40,21 +48,28 @@ uv pip install -e .
40
48
 
41
49
  ### Install from GitHub
42
50
 
51
+ `uv add steve-cli`
52
+
53
+ OR
54
+
43
55
  ```bash
44
56
  uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
45
57
  ```
46
58
 
59
+
60
+
47
61
  ## Usage
48
62
 
49
63
  ### Run a job
50
64
 
51
65
  ```bash
52
66
  steve extract-data
53
- steve transform-data
67
+ steve transform-data
54
68
  steve validate-data
55
69
  ```
56
70
 
57
71
  This will:
72
+
58
73
  1. Look for `jobs.yaml` in the current directory
59
74
  2. Find the specified job by name
60
75
  3. Set all environment variables from the job's `env` section
@@ -94,7 +109,7 @@ jobs:
94
109
  env:
95
110
  SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
96
111
  OUTPUT_PATH: "/data/raw/extracted_data.parquet"
97
-
112
+
98
113
  - name: transform-data
99
114
  dependsOn: ["extract-data"]
100
115
  command: ["uv", "run", "src/transform.py"]
@@ -104,6 +119,7 @@ jobs:
104
119
  ```
105
120
 
106
121
  Running `steve extract-data` will:
122
+
107
123
  1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
108
124
  2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
109
125
  3. Execute: `uv run src/extract.py`
@@ -141,3 +157,14 @@ pytest
141
157
  black steve_cli/
142
158
  isort steve_cli/
143
159
  ```
160
+
161
+
162
+
163
+ # use cli
164
+ source .venv/bin/activate
165
+ steve
166
+
167
+ #
168
+
169
+ uv build
170
+ uv publish
@@ -14,21 +14,28 @@ uv pip install -e .
14
14
 
15
15
  ### Install from GitHub
16
16
 
17
+ `uv add steve-cli`
18
+
19
+ OR
20
+
17
21
  ```bash
18
22
  uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
19
23
  ```
20
24
 
25
+
26
+
21
27
  ## Usage
22
28
 
23
29
  ### Run a job
24
30
 
25
31
  ```bash
26
32
  steve extract-data
27
- steve transform-data
33
+ steve transform-data
28
34
  steve validate-data
29
35
  ```
30
36
 
31
37
  This will:
38
+
32
39
  1. Look for `jobs.yaml` in the current directory
33
40
  2. Find the specified job by name
34
41
  3. Set all environment variables from the job's `env` section
@@ -68,7 +75,7 @@ jobs:
68
75
  env:
69
76
  SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
70
77
  OUTPUT_PATH: "/data/raw/extracted_data.parquet"
71
-
78
+
72
79
  - name: transform-data
73
80
  dependsOn: ["extract-data"]
74
81
  command: ["uv", "run", "src/transform.py"]
@@ -78,6 +85,7 @@ jobs:
78
85
  ```
79
86
 
80
87
  Running `steve extract-data` will:
88
+
81
89
  1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
82
90
  2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
83
91
  3. Execute: `uv run src/extract.py`
@@ -114,4 +122,15 @@ pytest
114
122
  # Format code
115
123
  black steve_cli/
116
124
  isort steve_cli/
117
- ```
125
+ ```
126
+
127
+
128
+
129
+ # use cli
130
+ source .venv/bin/activate
131
+ steve
132
+
133
+ #
134
+
135
+ uv build
136
+ uv publish
@@ -1,7 +1,11 @@
1
1
 
2
+ [build-system]
3
+ requires = ["setuptools>=61.0", "wheel"]
4
+ build-backend = "setuptools.build_meta"
5
+
2
6
  [project]
3
7
  name = "steve-cli"
4
- version = "0.1.0"
8
+ version = "0.3.0"
5
9
  description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
6
10
  readme = "README.md"
7
11
  license = {text = "MIT"}
@@ -19,8 +23,10 @@ classifiers = [
19
23
  ]
20
24
  requires-python = ">=3.8"
21
25
  dependencies = [
26
+ "boto3>=1.37.38",
22
27
  "click>=8.0.0",
23
28
  "pyyaml>=6.0",
29
+ "questionary>=2.0.0",
24
30
  ]
25
31
 
26
32
  [project.optional-dependencies]
@@ -29,6 +35,14 @@ dev = [
29
35
  "black>=22.0",
30
36
  "isort>=5.0",
31
37
  ]
38
+ polars = [
39
+ "polars>=1.8.2",
40
+ "pyarrow>=17.0.0",
41
+ ]
42
+ pandas = [
43
+ "pandas>=2.0.3",
44
+ "pyarrow>=17.0.0",
45
+ ]
32
46
 
33
47
  [project.scripts]
34
48
  steve = "steve_cli.cli:main"
@@ -39,9 +53,13 @@ Repository = "https://github.com/7frank/ds-steve-cli"
39
53
 
40
54
 
41
55
 
56
+ [tool.setuptools.packages.find]
57
+ where = ["."]
58
+ include = ["steve_cli*"]
59
+
42
60
  [tool.black]
43
61
  line-length = 88
44
62
  target-version = ['py38']
45
63
 
46
64
  [tool.isort]
47
- profile = "black"
65
+ profile = "black"
@@ -3,9 +3,9 @@
3
3
  Steve CLI - A simple tool to run jobs from jobs.yaml with proper environment setup.
4
4
 
5
5
  Usage:
6
- steve <job-name> Run a job with its environment variables
7
- steve ls List all available jobs
8
- steve help Show this help message
6
+ steve jobs ls List all available jobs
7
+ steve jobs run <job-name> Run a job with its environment variables
8
+ steve setup all Decrypt SOPS-encoded .env files
9
9
  """
10
10
 
11
11
  import os
@@ -15,90 +15,74 @@ from pathlib import Path
15
15
  from typing import Dict, List, Any, Optional
16
16
 
17
17
  import click
18
+ import questionary
18
19
  import yaml
19
20
 
20
21
 
21
22
  class JobsConfig:
22
- """Handle parsing and querying of jobs.yaml configuration."""
23
-
24
23
  def __init__(self, jobs_file: Optional[Path] = None):
25
24
  self.jobs_file = jobs_file or Path.cwd() / "jobs.yaml"
26
25
  self.jobs: List[Dict[str, Any]] = []
27
26
  self._load_jobs()
28
-
27
+
29
28
  def _load_jobs(self) -> None:
30
- """Load and parse the jobs.yaml file."""
31
29
  try:
32
30
  if not self.jobs_file.exists():
33
31
  click.echo(f"❌ Error: jobs.yaml not found at {self.jobs_file}", err=True)
34
32
  click.echo("Make sure you're in a directory with a jobs.yaml file", err=True)
35
33
  sys.exit(1)
36
-
34
+
37
35
  with open(self.jobs_file, 'r') as f:
38
36
  data = yaml.safe_load(f)
39
-
37
+
40
38
  if not data or 'jobs' not in data:
41
- click.echo(f"❌ Error: Invalid jobs.yaml format. Expected 'jobs' key at root", err=True)
39
+ click.echo("❌ Error: Invalid jobs.yaml format. Expected 'jobs' key at root", err=True)
42
40
  sys.exit(1)
43
-
41
+
44
42
  self.jobs = data['jobs']
45
-
43
+
46
44
  except yaml.YAMLError as e:
47
45
  click.echo(f"❌ Error parsing jobs.yaml: {e}", err=True)
48
46
  sys.exit(1)
49
47
  except Exception as e:
50
48
  click.echo(f"❌ Error reading jobs.yaml: {e}", err=True)
51
49
  sys.exit(1)
52
-
50
+
53
51
  def get_job(self, job_name: str) -> Optional[Dict[str, Any]]:
54
- """Get a job by name."""
55
52
  for job in self.jobs:
56
53
  if job.get('name') == job_name:
57
54
  return job
58
55
  return None
59
-
56
+
60
57
  def list_jobs(self) -> List[str]:
61
- """Get list of all job names."""
62
58
  return [job.get('name', 'unnamed') for job in self.jobs]
63
59
 
64
60
 
65
61
  def run_job_command(job: Dict[str, Any]) -> int:
66
- """Run a job's command with its environment variables."""
67
- # Get command
68
62
  command = job.get('command', [])
69
63
  if not command:
70
64
  click.echo(f"❌ Error: No command specified for job '{job.get('name')}'", err=True)
71
65
  return 1
72
-
73
- # Prepare environment
66
+
74
67
  env = os.environ.copy()
75
68
  job_env = job.get('env', {})
76
-
77
- # Convert all env values to strings and merge
78
69
  for key, value in job_env.items():
79
70
  env[key] = str(value)
80
-
81
- # Show what we're running
71
+
82
72
  job_name = job.get('name', 'unnamed')
83
73
  click.echo(f"🚀 Running job: {click.style(job_name, fg='blue', bold=True)}")
84
74
  click.echo(f"📝 Command: {click.style(' '.join(command), fg='cyan')}")
85
-
75
+
86
76
  if job_env:
87
- click.echo(f"🌍 Environment variables:")
77
+ click.echo("🌍 Environment variables:")
88
78
  for key, value in job_env.items():
89
79
  click.echo(f" {click.style(key, fg='green')}={click.style(str(value), fg='yellow')}")
90
-
91
- click.echo() # Empty line for readability
92
-
80
+
81
+ click.echo()
82
+
93
83
  try:
94
- # Run the command
95
- result = subprocess.run(
96
- command,
97
- env=env,
98
- cwd=Path.cwd(),
99
- )
84
+ result = subprocess.run(command, env=env, cwd=Path.cwd())
100
85
  return result.returncode
101
-
102
86
  except FileNotFoundError:
103
87
  click.echo(f"❌ Error: Command not found: {command[0]}", err=True)
104
88
  return 127
@@ -107,100 +91,147 @@ def run_job_command(job: Dict[str, Any]) -> int:
107
91
  return 1
108
92
 
109
93
 
110
- @click.group(invoke_without_command=True)
111
- @click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
94
+ @click.group()
95
+ def main():
96
+ """Steve CLI - Run jobs and manage environment setup."""
97
+ pass
98
+
99
+
100
+ @main.group(invoke_without_command=True)
101
+ @click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
112
102
  help='Path to jobs.yaml file (default: ./jobs.yaml)')
113
- @click.argument('job_name', required=False)
114
103
  @click.pass_context
115
- def main(ctx: click.Context, jobs_file: Optional[Path], job_name: Optional[str]):
116
- """Steve CLI - Run jobs from jobs.yaml with proper environment setup."""
117
-
118
- # Handle the case where a job name is provided directly
119
- if job_name:
120
- config = JobsConfig(jobs_file)
121
-
122
- # Special cases for built-in commands
123
- if job_name in ['ls', 'list']:
124
- list_jobs_command(config)
125
- return
126
- elif job_name in ['help', '--help', '-h']:
127
- click.echo(ctx.get_help())
128
- return
129
-
130
- # Try to run the job
131
- job = config.get_job(job_name)
132
- if not job:
133
- available_jobs = config.list_jobs()
134
- click.echo(f"❌ Error: Job '{job_name}' not found", err=True)
135
- if available_jobs:
136
- click.echo(f"Available jobs: {', '.join(available_jobs)}", err=True)
137
- else:
138
- click.echo("No jobs found in jobs.yaml", err=True)
139
- sys.exit(1)
140
-
141
- exit_code = run_job_command(job)
142
- sys.exit(exit_code)
143
-
144
- # If no job name provided, show help
104
+ def jobs(ctx: click.Context, jobs_file: Optional[Path]):
105
+ """Manage and run jobs from jobs.yaml."""
106
+ if ctx.invoked_subcommand is not None:
107
+ return
108
+ config = JobsConfig(jobs_file)
109
+ job_names = config.list_jobs()
110
+ if not job_names:
111
+ click.echo("❌ No jobs found in jobs.yaml")
112
+ return
145
113
  click.echo(ctx.get_help())
114
+ click.echo()
115
+ choice = questionary.select(
116
+ "Select a job to run:",
117
+ choices=job_names,
118
+ ).ask()
119
+ if choice is None:
120
+ sys.exit(0)
121
+ job = config.get_job(choice)
122
+ sys.exit(run_job_command(job))
146
123
 
147
124
 
148
- @main.command('ls')
125
+ @jobs.command('ls')
149
126
  @click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
150
127
  help='Path to jobs.yaml file (default: ./jobs.yaml)')
151
- def list_command(jobs_file: Optional[Path]):
128
+ def jobs_list(jobs_file: Optional[Path]):
152
129
  """List all available jobs."""
153
130
  config = JobsConfig(jobs_file)
154
- list_jobs_command(config)
155
-
131
+ job_names = config.list_jobs()
156
132
 
157
- def list_jobs_command(config: JobsConfig):
158
- """Show list of available jobs with details."""
159
- jobs = config.list_jobs()
160
-
161
- if not jobs:
133
+ if not job_names:
162
134
  click.echo("❌ No jobs found in jobs.yaml")
163
135
  return
164
-
136
+
165
137
  click.echo(f"📋 Available jobs in {click.style(str(config.jobs_file), fg='cyan')}:")
166
138
  click.echo()
167
-
139
+
168
140
  for job_data in config.jobs:
169
141
  name = job_data.get('name', 'unnamed')
170
142
  command = job_data.get('command', [])
171
143
  cron = job_data.get('cron')
172
144
  depends_on = job_data.get('dependsOn', [])
173
145
  env_vars = job_data.get('env', {})
174
-
175
- # Job name
146
+
176
147
  click.echo(f" {click.style(name, fg='blue', bold=True)}")
177
-
178
- # Command
179
148
  if command:
180
149
  click.echo(f" Command: {click.style(' '.join(command), fg='cyan')}")
181
-
182
- # Schedule
183
150
  if cron:
184
151
  click.echo(f" Schedule: {click.style(cron, fg='yellow')}")
185
-
186
- # Dependencies
187
152
  if depends_on:
188
153
  click.echo(f" Depends on: {click.style(', '.join(depends_on), fg='magenta')}")
189
-
190
- # Environment variables
191
154
  if env_vars:
192
155
  click.echo(f" Environment: {click.style(f'{len(env_vars)} variables', fg='green')}")
193
156
  for key, value in env_vars.items():
194
157
  click.echo(f" {key}={value}")
195
-
196
- click.echo() # Empty line between jobs
158
+ click.echo()
159
+
160
+
161
+ @jobs.command('run')
162
+ @click.argument('job_name')
163
+ @click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
164
+ help='Path to jobs.yaml file (default: ./jobs.yaml)')
165
+ def jobs_run(job_name: str, jobs_file: Optional[Path]):
166
+ """Run a job by name."""
167
+ config = JobsConfig(jobs_file)
168
+ job = config.get_job(job_name)
169
+ if not job:
170
+ available = config.list_jobs()
171
+ click.echo(f"❌ Error: Job '{job_name}' not found", err=True)
172
+ if available:
173
+ click.echo(f"Available jobs: {', '.join(available)}", err=True)
174
+ sys.exit(1)
175
+ sys.exit(run_job_command(job))
176
+
177
+
178
+ SETUP_IGNORE = [".env.example", ".env.template"]
179
+
180
+
181
+ @main.group()
182
+ def setup():
183
+ """Setup commands for environment configuration."""
184
+ pass
185
+
186
+
187
+ @setup.command("env")
188
+ def setup_env():
189
+ """Decrypt SOPS-encoded .env files in current directory and write plaintext .env files."""
190
+ cwd = Path.cwd()
191
+ found: List[Path] = []
192
+ for pattern in ["*.enc.env", "*.encoded.env"]:
193
+ found.extend(sorted(cwd.glob(pattern)))
194
+
195
+ found = [f for f in found if f.name not in SETUP_IGNORE]
196
+
197
+ if not found:
198
+ click.echo("No *.enc.env or *.encoded.env files found.")
199
+ return
200
+
201
+ for enc_file in found:
202
+ click.echo(f"\n📄 Found: {click.style(enc_file.name, fg='blue', bold=True)}")
203
+
204
+ content = enc_file.read_text()
205
+ if "ENC[" not in content and "sops:" not in content:
206
+ click.echo(" ⚠️ Not SOPS encrypted, skipping.")
207
+ continue
208
+
209
+ result = subprocess.run(
210
+ ["sops", "-d", str(enc_file)],
211
+ capture_output=True,
212
+ text=True,
213
+ )
214
+
215
+ if result.returncode != 0:
216
+ click.echo(f" ❌ Decryption failed: {result.stderr.strip()}", err=True)
217
+ continue
218
+
219
+ keys = []
220
+ for line in result.stdout.splitlines():
221
+ line = line.strip()
222
+ if line and not line.startswith("#") and "=" in line:
223
+ keys.append(line.split("=", 1)[0])
224
+
225
+ if keys:
226
+ click.echo(f" 🔑 Keys: {click.style(', '.join(keys), fg='green')}")
197
227
 
228
+ stem = enc_file.name.replace(".encoded.env", "").replace(".enc.env", "")
229
+ out_file = cwd / f"{stem}.env"
230
+ out_file.write_text(result.stdout)
198
231
 
199
- @main.command('help')
200
- def help_command():
201
- """Show help information."""
202
- click.echo(__doc__)
232
+ click.echo(f" ✅ Decrypted -> {click.style(out_file.name, fg='cyan')}")
233
+ click.echo(f" 💡 Run: {click.style(f'source {out_file.name}', fg='yellow')}")
203
234
 
204
235
 
205
236
  if __name__ == '__main__':
206
- main()
237
+ main()
@@ -0,0 +1,158 @@
1
+ import os
2
+ from typing import Protocol, List
3
+ from pathlib import PurePosixPath
4
+ import sys
5
+ import boto3
6
+ from botocore.client import Config
7
+ from botocore.exceptions import ClientError, EndpointConnectionError
8
+
9
+
10
+ class Storage(Protocol):
11
+ def put_file(self, local_path: str, path: str) -> None: ...
12
+ def get_file(self, path: str, local_path: str) -> None: ...
13
+ def put_bytes(self, data: bytes, path: str) -> None: ...
14
+ def get_bytes(self, path: str) -> bytes: ...
15
+ def list(self, prefix: str = "") -> List[str]: ...
16
+
17
+
18
+ class S3Storage:
19
+ def __init__(self, tier: str = "bronze"):
20
+ self.tier = tier.lower()
21
+
22
+ if self.tier == "bronze":
23
+ access_key = os.getenv("BRONZE_ACCESS_KEY")
24
+ secret_key = os.getenv("BRONZE_SECRET_KEY")
25
+ self.bucket = os.getenv("BRONZE_BUCKET")
26
+ required_vars = ["BRONZE_ACCESS_KEY", "BRONZE_SECRET_KEY", "BRONZE_BUCKET"]
27
+ elif self.tier == "silver":
28
+ access_key = os.getenv("SILVER_ACCESS_KEY")
29
+ secret_key = os.getenv("SILVER_SECRET_KEY")
30
+ self.bucket = os.getenv("SILVER_BUCKET")
31
+ required_vars = ["SILVER_ACCESS_KEY", "SILVER_SECRET_KEY", "SILVER_BUCKET"]
32
+ else:
33
+ access_key = os.getenv("GOLD_ACCESS_KEY")
34
+ secret_key = os.getenv("GOLD_SECRET_KEY")
35
+ self.bucket = os.getenv("GOLD_BUCKET")
36
+ required_vars = ["GOLD_ACCESS_KEY", "GOLD_SECRET_KEY", "GOLD_BUCKET"]
37
+
38
+ missing_vars = [var for var in required_vars if not os.getenv(var)]
39
+ if missing_vars:
40
+ raise EnvironmentError(
41
+ f"Missing required environment variables for {self.tier} tier: {', '.join(missing_vars)}"
42
+ )
43
+
44
+ endpoint = os.getenv("S3_ENDPOINT", "http://localhost:9000")
45
+
46
+ self.client = boto3.client(
47
+ "s3",
48
+ endpoint_url=endpoint,
49
+ aws_access_key_id=access_key,
50
+ aws_secret_access_key=secret_key,
51
+ config=Config(signature_version="s3v4"),
52
+ region_name="us-east-1",
53
+ )
54
+
55
+ self.endpoint = endpoint
56
+
57
+ def _key(self, path: str) -> str:
58
+ return str(PurePosixPath(path).as_posix().lstrip("/"))
59
+
60
+ def _get_absolute_url(self, path: str) -> str:
61
+ key = self._key(path)
62
+ return f"{self.endpoint}/{self.bucket}/{key}"
63
+
64
+ def put_file(self, local_path: str, path: str) -> None:
65
+ absolute_url = self._get_absolute_url(path)
66
+ try:
67
+ self.client.upload_file(local_path, self.bucket, self._key(path))
68
+ print(f"🔗 File uploaded to: {absolute_url}")
69
+ except EndpointConnectionError:
70
+ print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
71
+ sys.exit(1)
72
+ except ClientError as e:
73
+ error_code = e.response['Error']['Code']
74
+ if error_code == 'AccessDenied':
75
+ print(f"❌ Access denied when uploading to: {path}", file=sys.stderr)
76
+ elif error_code == 'NoSuchBucket':
77
+ print(f"❌ Bucket not found: {self.bucket}", file=sys.stderr)
78
+ else:
79
+ print(f"❌ Storage error when uploading to: {path}: {e}", file=sys.stderr)
80
+ sys.exit(1)
81
+
82
+ def get_file(self, path: str, local_path: str) -> None:
83
+ os.makedirs(os.path.dirname(local_path), exist_ok=True)
84
+ absolute_url = self._get_absolute_url(path)
85
+ try:
86
+ self.client.download_file(self.bucket, self._key(path), local_path)
87
+ print(f"📥 File downloaded from: {absolute_url}")
88
+ except EndpointConnectionError:
89
+ print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
90
+ sys.exit(1)
91
+ except ClientError as e:
92
+ error_code = e.response['Error']['Code']
93
+ if error_code == 'NoSuchKey':
94
+ print(f"❌ File not found: {path}", file=sys.stderr)
95
+ elif error_code == 'AccessDenied':
96
+ print(f"❌ Access denied when reading: {path}", file=sys.stderr)
97
+ else:
98
+ print(f"❌ Storage error when reading: {path}: {e}", file=sys.stderr)
99
+ sys.exit(1)
100
+
101
+ def put_bytes(self, data: bytes, path: str) -> None:
102
+ absolute_url = self._get_absolute_url(path)
103
+ try:
104
+ self.client.put_object(
105
+ Bucket=self.bucket,
106
+ Key=self._key(path),
107
+ Body=data
108
+ )
109
+ print(f"💾 Data stored at: {absolute_url}")
110
+ except EndpointConnectionError:
111
+ print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
112
+ sys.exit(1)
113
+ except ClientError as e:
114
+ error_code = e.response['Error']['Code']
115
+ if error_code == 'AccessDenied':
116
+ print(f"❌ Access denied when writing to: {path}", file=sys.stderr)
117
+ elif error_code == 'NoSuchBucket':
118
+ print(f"❌ Bucket not found: {self.bucket}", file=sys.stderr)
119
+ else:
120
+ print(f"❌ Storage error when writing to: {path}: {e}", file=sys.stderr)
121
+ sys.exit(1)
122
+
123
+ def get_bytes(self, path: str) -> bytes:
124
+ absolute_url = self._get_absolute_url(path)
125
+ try:
126
+ obj = self.client.get_object(
127
+ Bucket=self.bucket,
128
+ Key=self._key(path)
129
+ )
130
+ print(f"📥 Data retrieved from: {absolute_url}")
131
+ return obj["Body"].read()
132
+ except EndpointConnectionError:
133
+ print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
134
+ sys.exit(1)
135
+ except ClientError as e:
136
+ error_code = e.response['Error']['Code']
137
+ if error_code == 'NoSuchKey':
138
+ print(f"❌ File not found: {path}", file=sys.stderr)
139
+ elif error_code == 'AccessDenied':
140
+ print(f"❌ Access denied when reading: {path}", file=sys.stderr)
141
+ else:
142
+ print(f"❌ Storage error when reading: {path}: {e}", file=sys.stderr)
143
+ sys.exit(1)
144
+
145
+ def list(self, prefix: str = "") -> List[str]:
146
+ resp = self.client.list_objects_v2(
147
+ Bucket=self.bucket,
148
+ Prefix=self._key(prefix)
149
+ )
150
+
151
+ return [
152
+ obj["Key"]
153
+ for obj in resp.get("Contents", [])
154
+ ]
155
+
156
+
157
+ def get_storage(tier: str = "bronze") -> Storage:
158
+ return S3Storage(tier=tier)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 0.1.0
3
+ Version: 0.3.0
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -17,12 +17,20 @@ Classifier: Programming Language :: Python :: 3.11
17
17
  Classifier: Programming Language :: Python :: 3.12
18
18
  Requires-Python: >=3.8
19
19
  Description-Content-Type: text/markdown
20
+ Requires-Dist: boto3>=1.37.38
20
21
  Requires-Dist: click>=8.0.0
21
22
  Requires-Dist: pyyaml>=6.0
23
+ Requires-Dist: questionary>=2.0.0
22
24
  Provides-Extra: dev
23
25
  Requires-Dist: pytest>=7.0; extra == "dev"
24
26
  Requires-Dist: black>=22.0; extra == "dev"
25
27
  Requires-Dist: isort>=5.0; extra == "dev"
28
+ Provides-Extra: polars
29
+ Requires-Dist: polars>=1.8.2; extra == "polars"
30
+ Requires-Dist: pyarrow>=17.0.0; extra == "polars"
31
+ Provides-Extra: pandas
32
+ Requires-Dist: pandas>=2.0.3; extra == "pandas"
33
+ Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
26
34
 
27
35
  # Steve CLI
28
36
 
@@ -40,21 +48,28 @@ uv pip install -e .
40
48
 
41
49
  ### Install from GitHub
42
50
 
51
+ `uv add steve-cli`
52
+
53
+ OR
54
+
43
55
  ```bash
44
56
  uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
45
57
  ```
46
58
 
59
+
60
+
47
61
  ## Usage
48
62
 
49
63
  ### Run a job
50
64
 
51
65
  ```bash
52
66
  steve extract-data
53
- steve transform-data
67
+ steve transform-data
54
68
  steve validate-data
55
69
  ```
56
70
 
57
71
  This will:
72
+
58
73
  1. Look for `jobs.yaml` in the current directory
59
74
  2. Find the specified job by name
60
75
  3. Set all environment variables from the job's `env` section
@@ -94,7 +109,7 @@ jobs:
94
109
  env:
95
110
  SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
96
111
  OUTPUT_PATH: "/data/raw/extracted_data.parquet"
97
-
112
+
98
113
  - name: transform-data
99
114
  dependsOn: ["extract-data"]
100
115
  command: ["uv", "run", "src/transform.py"]
@@ -104,6 +119,7 @@ jobs:
104
119
  ```
105
120
 
106
121
  Running `steve extract-data` will:
122
+
107
123
  1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
108
124
  2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
109
125
  3. Execute: `uv run src/extract.py`
@@ -141,3 +157,14 @@ pytest
141
157
  black steve_cli/
142
158
  isort steve_cli/
143
159
  ```
160
+
161
+
162
+
163
+ # use cli
164
+ source .venv/bin/activate
165
+ steve
166
+
167
+ #
168
+
169
+ uv build
170
+ uv publish
@@ -2,6 +2,7 @@ README.md
2
2
  pyproject.toml
3
3
  steve_cli/__init__.py
4
4
  steve_cli/cli.py
5
+ steve_cli/storage.py
5
6
  steve_cli.egg-info/PKG-INFO
6
7
  steve_cli.egg-info/SOURCES.txt
7
8
  steve_cli.egg-info/dependency_links.txt
@@ -0,0 +1,17 @@
1
+ boto3>=1.37.38
2
+ click>=8.0.0
3
+ pyyaml>=6.0
4
+ questionary>=2.0.0
5
+
6
+ [dev]
7
+ pytest>=7.0
8
+ black>=22.0
9
+ isort>=5.0
10
+
11
+ [pandas]
12
+ pandas>=2.0.3
13
+ pyarrow>=17.0.0
14
+
15
+ [polars]
16
+ polars>=1.8.2
17
+ pyarrow>=17.0.0
@@ -1,7 +0,0 @@
1
- click>=8.0.0
2
- pyyaml>=6.0
3
-
4
- [dev]
5
- pytest>=7.0
6
- black>=22.0
7
- isort>=5.0
File without changes