steve-cli 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -17,12 +17,19 @@ Classifier: Programming Language :: Python :: 3.11
17
17
  Classifier: Programming Language :: Python :: 3.12
18
18
  Requires-Python: >=3.8
19
19
  Description-Content-Type: text/markdown
20
+ Requires-Dist: boto3>=1.37.38
20
21
  Requires-Dist: click>=8.0.0
21
22
  Requires-Dist: pyyaml>=6.0
22
23
  Provides-Extra: dev
23
24
  Requires-Dist: pytest>=7.0; extra == "dev"
24
25
  Requires-Dist: black>=22.0; extra == "dev"
25
26
  Requires-Dist: isort>=5.0; extra == "dev"
27
+ Provides-Extra: polars
28
+ Requires-Dist: polars>=1.8.2; extra == "polars"
29
+ Requires-Dist: pyarrow>=17.0.0; extra == "polars"
30
+ Provides-Extra: pandas
31
+ Requires-Dist: pandas>=2.0.3; extra == "pandas"
32
+ Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
26
33
 
27
34
  # Steve CLI
28
35
 
@@ -40,21 +47,28 @@ uv pip install -e .
40
47
 
41
48
  ### Install from GitHub
42
49
 
50
+ `uv add steve-cli`
51
+
52
+ OR
53
+
43
54
  ```bash
44
55
  uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
45
56
  ```
46
57
 
58
+
59
+
47
60
  ## Usage
48
61
 
49
62
  ### Run a job
50
63
 
51
64
  ```bash
52
65
  steve extract-data
53
- steve transform-data
66
+ steve transform-data
54
67
  steve validate-data
55
68
  ```
56
69
 
57
70
  This will:
71
+
58
72
  1. Look for `jobs.yaml` in the current directory
59
73
  2. Find the specified job by name
60
74
  3. Set all environment variables from the job's `env` section
@@ -94,7 +108,7 @@ jobs:
94
108
  env:
95
109
  SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
96
110
  OUTPUT_PATH: "/data/raw/extracted_data.parquet"
97
-
111
+
98
112
  - name: transform-data
99
113
  dependsOn: ["extract-data"]
100
114
  command: ["uv", "run", "src/transform.py"]
@@ -104,6 +118,7 @@ jobs:
104
118
  ```
105
119
 
106
120
  Running `steve extract-data` will:
121
+
107
122
  1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
108
123
  2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
109
124
  3. Execute: `uv run src/extract.py`
@@ -141,3 +156,8 @@ pytest
141
156
  black steve_cli/
142
157
  isort steve_cli/
143
158
  ```
159
+
160
+ #
161
+
162
+ uv build
163
+ uv publish
@@ -14,21 +14,28 @@ uv pip install -e .
14
14
 
15
15
  ### Install from GitHub
16
16
 
17
+ `uv add steve-cli`
18
+
19
+ OR
20
+
17
21
  ```bash
18
22
  uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
19
23
  ```
20
24
 
25
+
26
+
21
27
  ## Usage
22
28
 
23
29
  ### Run a job
24
30
 
25
31
  ```bash
26
32
  steve extract-data
27
- steve transform-data
33
+ steve transform-data
28
34
  steve validate-data
29
35
  ```
30
36
 
31
37
  This will:
38
+
32
39
  1. Look for `jobs.yaml` in the current directory
33
40
  2. Find the specified job by name
34
41
  3. Set all environment variables from the job's `env` section
@@ -68,7 +75,7 @@ jobs:
68
75
  env:
69
76
  SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
70
77
  OUTPUT_PATH: "/data/raw/extracted_data.parquet"
71
-
78
+
72
79
  - name: transform-data
73
80
  dependsOn: ["extract-data"]
74
81
  command: ["uv", "run", "src/transform.py"]
@@ -78,6 +85,7 @@ jobs:
78
85
  ```
79
86
 
80
87
  Running `steve extract-data` will:
88
+
81
89
  1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
82
90
  2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
83
91
  3. Execute: `uv run src/extract.py`
@@ -114,4 +122,9 @@ pytest
114
122
  # Format code
115
123
  black steve_cli/
116
124
  isort steve_cli/
117
- ```
125
+ ```
126
+
127
+ #
128
+
129
+ uv build
130
+ uv publish
@@ -1,7 +1,7 @@
1
1
 
2
2
  [project]
3
3
  name = "steve-cli"
4
- version = "0.1.0"
4
+ version = "0.2.0"
5
5
  description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
6
6
  readme = "README.md"
7
7
  license = {text = "MIT"}
@@ -19,6 +19,7 @@ classifiers = [
19
19
  ]
20
20
  requires-python = ">=3.8"
21
21
  dependencies = [
22
+ "boto3>=1.37.38",
22
23
  "click>=8.0.0",
23
24
  "pyyaml>=6.0",
24
25
  ]
@@ -29,6 +30,14 @@ dev = [
29
30
  "black>=22.0",
30
31
  "isort>=5.0",
31
32
  ]
33
+ polars = [
34
+ "polars>=1.8.2",
35
+ "pyarrow>=17.0.0",
36
+ ]
37
+ pandas = [
38
+ "pandas>=2.0.3",
39
+ "pyarrow>=17.0.0",
40
+ ]
32
41
 
33
42
  [project.scripts]
34
43
  steve = "steve_cli.cli:main"
@@ -39,9 +48,13 @@ Repository = "https://github.com/7frank/ds-steve-cli"
39
48
 
40
49
 
41
50
 
51
+ [tool.setuptools.packages.find]
52
+ where = ["."]
53
+ include = ["steve_cli*"]
54
+
42
55
  [tool.black]
43
56
  line-length = 88
44
57
  target-version = ['py38']
45
58
 
46
59
  [tool.isort]
47
- profile = "black"
60
+ profile = "black"
@@ -3,9 +3,9 @@
3
3
  Steve CLI - A simple tool to run jobs from jobs.yaml with proper environment setup.
4
4
 
5
5
  Usage:
6
- steve <job-name> Run a job with its environment variables
7
- steve ls List all available jobs
8
- steve help Show this help message
6
+ steve jobs ls List all available jobs
7
+ steve jobs run <job-name> Run a job with its environment variables
8
+ steve setup all Decrypt SOPS-encoded .env files
9
9
  """
10
10
 
11
11
  import os
@@ -19,86 +19,69 @@ import yaml
19
19
 
20
20
 
21
21
  class JobsConfig:
22
- """Handle parsing and querying of jobs.yaml configuration."""
23
-
24
22
  def __init__(self, jobs_file: Optional[Path] = None):
25
23
  self.jobs_file = jobs_file or Path.cwd() / "jobs.yaml"
26
24
  self.jobs: List[Dict[str, Any]] = []
27
25
  self._load_jobs()
28
-
26
+
29
27
  def _load_jobs(self) -> None:
30
- """Load and parse the jobs.yaml file."""
31
28
  try:
32
29
  if not self.jobs_file.exists():
33
30
  click.echo(f"❌ Error: jobs.yaml not found at {self.jobs_file}", err=True)
34
31
  click.echo("Make sure you're in a directory with a jobs.yaml file", err=True)
35
32
  sys.exit(1)
36
-
33
+
37
34
  with open(self.jobs_file, 'r') as f:
38
35
  data = yaml.safe_load(f)
39
-
36
+
40
37
  if not data or 'jobs' not in data:
41
- click.echo(f"❌ Error: Invalid jobs.yaml format. Expected 'jobs' key at root", err=True)
38
+ click.echo("❌ Error: Invalid jobs.yaml format. Expected 'jobs' key at root", err=True)
42
39
  sys.exit(1)
43
-
40
+
44
41
  self.jobs = data['jobs']
45
-
42
+
46
43
  except yaml.YAMLError as e:
47
44
  click.echo(f"❌ Error parsing jobs.yaml: {e}", err=True)
48
45
  sys.exit(1)
49
46
  except Exception as e:
50
47
  click.echo(f"❌ Error reading jobs.yaml: {e}", err=True)
51
48
  sys.exit(1)
52
-
49
+
53
50
  def get_job(self, job_name: str) -> Optional[Dict[str, Any]]:
54
- """Get a job by name."""
55
51
  for job in self.jobs:
56
52
  if job.get('name') == job_name:
57
53
  return job
58
54
  return None
59
-
55
+
60
56
  def list_jobs(self) -> List[str]:
61
- """Get list of all job names."""
62
57
  return [job.get('name', 'unnamed') for job in self.jobs]
63
58
 
64
59
 
65
60
  def run_job_command(job: Dict[str, Any]) -> int:
66
- """Run a job's command with its environment variables."""
67
- # Get command
68
61
  command = job.get('command', [])
69
62
  if not command:
70
63
  click.echo(f"❌ Error: No command specified for job '{job.get('name')}'", err=True)
71
64
  return 1
72
-
73
- # Prepare environment
65
+
74
66
  env = os.environ.copy()
75
67
  job_env = job.get('env', {})
76
-
77
- # Convert all env values to strings and merge
78
68
  for key, value in job_env.items():
79
69
  env[key] = str(value)
80
-
81
- # Show what we're running
70
+
82
71
  job_name = job.get('name', 'unnamed')
83
72
  click.echo(f"🚀 Running job: {click.style(job_name, fg='blue', bold=True)}")
84
73
  click.echo(f"📝 Command: {click.style(' '.join(command), fg='cyan')}")
85
-
74
+
86
75
  if job_env:
87
- click.echo(f"🌍 Environment variables:")
76
+ click.echo("🌍 Environment variables:")
88
77
  for key, value in job_env.items():
89
78
  click.echo(f" {click.style(key, fg='green')}={click.style(str(value), fg='yellow')}")
90
-
91
- click.echo() # Empty line for readability
92
-
79
+
80
+ click.echo()
81
+
93
82
  try:
94
- # Run the command
95
- result = subprocess.run(
96
- command,
97
- env=env,
98
- cwd=Path.cwd(),
99
- )
83
+ result = subprocess.run(command, env=env, cwd=Path.cwd())
100
84
  return result.returncode
101
-
102
85
  except FileNotFoundError:
103
86
  click.echo(f"❌ Error: Command not found: {command[0]}", err=True)
104
87
  return 127
@@ -107,100 +90,128 @@ def run_job_command(job: Dict[str, Any]) -> int:
107
90
  return 1
108
91
 
109
92
 
110
- @click.group(invoke_without_command=True)
111
- @click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
112
- help='Path to jobs.yaml file (default: ./jobs.yaml)')
113
- @click.argument('job_name', required=False)
114
- @click.pass_context
115
- def main(ctx: click.Context, jobs_file: Optional[Path], job_name: Optional[str]):
116
- """Steve CLI - Run jobs from jobs.yaml with proper environment setup."""
117
-
118
- # Handle the case where a job name is provided directly
119
- if job_name:
120
- config = JobsConfig(jobs_file)
121
-
122
- # Special cases for built-in commands
123
- if job_name in ['ls', 'list']:
124
- list_jobs_command(config)
125
- return
126
- elif job_name in ['help', '--help', '-h']:
127
- click.echo(ctx.get_help())
128
- return
129
-
130
- # Try to run the job
131
- job = config.get_job(job_name)
132
- if not job:
133
- available_jobs = config.list_jobs()
134
- click.echo(f"❌ Error: Job '{job_name}' not found", err=True)
135
- if available_jobs:
136
- click.echo(f"Available jobs: {', '.join(available_jobs)}", err=True)
137
- else:
138
- click.echo("No jobs found in jobs.yaml", err=True)
139
- sys.exit(1)
140
-
141
- exit_code = run_job_command(job)
142
- sys.exit(exit_code)
143
-
144
- # If no job name provided, show help
145
- click.echo(ctx.get_help())
93
+ @click.group()
94
+ def main():
95
+ """Steve CLI - Run jobs and manage environment setup."""
96
+ pass
97
+
146
98
 
99
+ @main.group()
100
+ def jobs():
101
+ """Manage and run jobs from jobs.yaml."""
102
+ pass
147
103
 
148
- @main.command('ls')
104
+
105
+ @jobs.command('ls')
149
106
  @click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
150
107
  help='Path to jobs.yaml file (default: ./jobs.yaml)')
151
- def list_command(jobs_file: Optional[Path]):
108
+ def jobs_list(jobs_file: Optional[Path]):
152
109
  """List all available jobs."""
153
110
  config = JobsConfig(jobs_file)
154
- list_jobs_command(config)
155
-
111
+ job_names = config.list_jobs()
156
112
 
157
- def list_jobs_command(config: JobsConfig):
158
- """Show list of available jobs with details."""
159
- jobs = config.list_jobs()
160
-
161
- if not jobs:
113
+ if not job_names:
162
114
  click.echo("❌ No jobs found in jobs.yaml")
163
115
  return
164
-
116
+
165
117
  click.echo(f"📋 Available jobs in {click.style(str(config.jobs_file), fg='cyan')}:")
166
118
  click.echo()
167
-
119
+
168
120
  for job_data in config.jobs:
169
121
  name = job_data.get('name', 'unnamed')
170
122
  command = job_data.get('command', [])
171
123
  cron = job_data.get('cron')
172
124
  depends_on = job_data.get('dependsOn', [])
173
125
  env_vars = job_data.get('env', {})
174
-
175
- # Job name
126
+
176
127
  click.echo(f" {click.style(name, fg='blue', bold=True)}")
177
-
178
- # Command
179
128
  if command:
180
129
  click.echo(f" Command: {click.style(' '.join(command), fg='cyan')}")
181
-
182
- # Schedule
183
130
  if cron:
184
131
  click.echo(f" Schedule: {click.style(cron, fg='yellow')}")
185
-
186
- # Dependencies
187
132
  if depends_on:
188
133
  click.echo(f" Depends on: {click.style(', '.join(depends_on), fg='magenta')}")
189
-
190
- # Environment variables
191
134
  if env_vars:
192
135
  click.echo(f" Environment: {click.style(f'{len(env_vars)} variables', fg='green')}")
193
136
  for key, value in env_vars.items():
194
137
  click.echo(f" {key}={value}")
195
-
196
- click.echo() # Empty line between jobs
138
+ click.echo()
139
+
140
+
141
+ @jobs.command('run')
142
+ @click.argument('job_name')
143
+ @click.option('--jobs-file', '-f', type=click.Path(exists=True, path_type=Path),
144
+ help='Path to jobs.yaml file (default: ./jobs.yaml)')
145
+ def jobs_run(job_name: str, jobs_file: Optional[Path]):
146
+ """Run a job by name."""
147
+ config = JobsConfig(jobs_file)
148
+ job = config.get_job(job_name)
149
+ if not job:
150
+ available = config.list_jobs()
151
+ click.echo(f"❌ Error: Job '{job_name}' not found", err=True)
152
+ if available:
153
+ click.echo(f"Available jobs: {', '.join(available)}", err=True)
154
+ sys.exit(1)
155
+ sys.exit(run_job_command(job))
156
+
157
+
158
+ SETUP_IGNORE = [".env.example", ".env.template"]
159
+
160
+
161
+ @main.group()
162
+ def setup():
163
+ """Setup commands for environment configuration."""
164
+ pass
165
+
166
+
167
+ @setup.command("all")
168
+ def setup_all():
169
+ """Decrypt SOPS-encoded .env files in current directory and write plaintext .env files."""
170
+ cwd = Path.cwd()
171
+ found: List[Path] = []
172
+ for pattern in ["*.enc.env", "*.encoded.env"]:
173
+ found.extend(sorted(cwd.glob(pattern)))
174
+
175
+ found = [f for f in found if f.name not in SETUP_IGNORE]
176
+
177
+ if not found:
178
+ click.echo("No *.enc.env or *.encoded.env files found.")
179
+ return
180
+
181
+ for enc_file in found:
182
+ click.echo(f"\n📄 Found: {click.style(enc_file.name, fg='blue', bold=True)}")
183
+
184
+ content = enc_file.read_text()
185
+ if "ENC[" not in content and "sops:" not in content:
186
+ click.echo(" ⚠️ Not SOPS encrypted, skipping.")
187
+ continue
188
+
189
+ result = subprocess.run(
190
+ ["sops", "-d", str(enc_file)],
191
+ capture_output=True,
192
+ text=True,
193
+ )
194
+
195
+ if result.returncode != 0:
196
+ click.echo(f" ❌ Decryption failed: {result.stderr.strip()}", err=True)
197
+ continue
198
+
199
+ keys = []
200
+ for line in result.stdout.splitlines():
201
+ line = line.strip()
202
+ if line and not line.startswith("#") and "=" in line:
203
+ keys.append(line.split("=", 1)[0])
204
+
205
+ if keys:
206
+ click.echo(f" 🔑 Keys: {click.style(', '.join(keys), fg='green')}")
197
207
 
208
+ stem = enc_file.name.replace(".encoded.env", "").replace(".enc.env", "")
209
+ out_file = cwd / f"{stem}.env"
210
+ out_file.write_text(result.stdout)
198
211
 
199
- @main.command('help')
200
- def help_command():
201
- """Show help information."""
202
- click.echo(__doc__)
212
+ click.echo(f" ✅ Decrypted -> {click.style(out_file.name, fg='cyan')}")
213
+ click.echo(f" 💡 Run: {click.style(f'source {out_file.name}', fg='yellow')}")
203
214
 
204
215
 
205
216
  if __name__ == '__main__':
206
- main()
217
+ main()
@@ -0,0 +1,158 @@
1
+ import os
2
+ from typing import Protocol, List
3
+ from pathlib import PurePosixPath
4
+ import sys
5
+ import boto3
6
+ from botocore.client import Config
7
+ from botocore.exceptions import ClientError, EndpointConnectionError
8
+
9
+
10
+ class Storage(Protocol):
11
+ def put_file(self, local_path: str, path: str) -> None: ...
12
+ def get_file(self, path: str, local_path: str) -> None: ...
13
+ def put_bytes(self, data: bytes, path: str) -> None: ...
14
+ def get_bytes(self, path: str) -> bytes: ...
15
+ def list(self, prefix: str = "") -> List[str]: ...
16
+
17
+
18
+ class S3Storage:
19
+ def __init__(self, tier: str = "bronze"):
20
+ self.tier = tier.lower()
21
+
22
+ if self.tier == "bronze":
23
+ access_key = os.getenv("BRONZE_ACCESS_KEY")
24
+ secret_key = os.getenv("BRONZE_SECRET_KEY")
25
+ self.bucket = os.getenv("BRONZE_BUCKET")
26
+ required_vars = ["BRONZE_ACCESS_KEY", "BRONZE_SECRET_KEY", "BRONZE_BUCKET"]
27
+ elif self.tier == "silver":
28
+ access_key = os.getenv("SILVER_ACCESS_KEY")
29
+ secret_key = os.getenv("SILVER_SECRET_KEY")
30
+ self.bucket = os.getenv("SILVER_BUCKET")
31
+ required_vars = ["SILVER_ACCESS_KEY", "SILVER_SECRET_KEY", "SILVER_BUCKET"]
32
+ else:
33
+ access_key = os.getenv("GOLD_ACCESS_KEY")
34
+ secret_key = os.getenv("GOLD_SECRET_KEY")
35
+ self.bucket = os.getenv("GOLD_BUCKET")
36
+ required_vars = ["GOLD_ACCESS_KEY", "GOLD_SECRET_KEY", "GOLD_BUCKET"]
37
+
38
+ missing_vars = [var for var in required_vars if not os.getenv(var)]
39
+ if missing_vars:
40
+ raise EnvironmentError(
41
+ f"Missing required environment variables for {self.tier} tier: {', '.join(missing_vars)}"
42
+ )
43
+
44
+ endpoint = os.getenv("S3_ENDPOINT", "http://localhost:9000")
45
+
46
+ self.client = boto3.client(
47
+ "s3",
48
+ endpoint_url=endpoint,
49
+ aws_access_key_id=access_key,
50
+ aws_secret_access_key=secret_key,
51
+ config=Config(signature_version="s3v4"),
52
+ region_name="us-east-1",
53
+ )
54
+
55
+ self.endpoint = endpoint
56
+
57
+ def _key(self, path: str) -> str:
58
+ return str(PurePosixPath(path).as_posix().lstrip("/"))
59
+
60
+ def _get_absolute_url(self, path: str) -> str:
61
+ key = self._key(path)
62
+ return f"{self.endpoint}/{self.bucket}/{key}"
63
+
64
+ def put_file(self, local_path: str, path: str) -> None:
65
+ absolute_url = self._get_absolute_url(path)
66
+ try:
67
+ self.client.upload_file(local_path, self.bucket, self._key(path))
68
+ print(f"🔗 File uploaded to: {absolute_url}")
69
+ except EndpointConnectionError:
70
+ print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
71
+ sys.exit(1)
72
+ except ClientError as e:
73
+ error_code = e.response['Error']['Code']
74
+ if error_code == 'AccessDenied':
75
+ print(f"❌ Access denied when uploading to: {path}", file=sys.stderr)
76
+ elif error_code == 'NoSuchBucket':
77
+ print(f"❌ Bucket not found: {self.bucket}", file=sys.stderr)
78
+ else:
79
+ print(f"❌ Storage error when uploading to: {path}: {e}", file=sys.stderr)
80
+ sys.exit(1)
81
+
82
+ def get_file(self, path: str, local_path: str) -> None:
83
+ os.makedirs(os.path.dirname(local_path), exist_ok=True)
84
+ absolute_url = self._get_absolute_url(path)
85
+ try:
86
+ self.client.download_file(self.bucket, self._key(path), local_path)
87
+ print(f"📥 File downloaded from: {absolute_url}")
88
+ except EndpointConnectionError:
89
+ print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
90
+ sys.exit(1)
91
+ except ClientError as e:
92
+ error_code = e.response['Error']['Code']
93
+ if error_code == 'NoSuchKey':
94
+ print(f"❌ File not found: {path}", file=sys.stderr)
95
+ elif error_code == 'AccessDenied':
96
+ print(f"❌ Access denied when reading: {path}", file=sys.stderr)
97
+ else:
98
+ print(f"❌ Storage error when reading: {path}: {e}", file=sys.stderr)
99
+ sys.exit(1)
100
+
101
+ def put_bytes(self, data: bytes, path: str) -> None:
102
+ absolute_url = self._get_absolute_url(path)
103
+ try:
104
+ self.client.put_object(
105
+ Bucket=self.bucket,
106
+ Key=self._key(path),
107
+ Body=data
108
+ )
109
+ print(f"💾 Data stored at: {absolute_url}")
110
+ except EndpointConnectionError:
111
+ print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
112
+ sys.exit(1)
113
+ except ClientError as e:
114
+ error_code = e.response['Error']['Code']
115
+ if error_code == 'AccessDenied':
116
+ print(f"❌ Access denied when writing to: {path}", file=sys.stderr)
117
+ elif error_code == 'NoSuchBucket':
118
+ print(f"❌ Bucket not found: {self.bucket}", file=sys.stderr)
119
+ else:
120
+ print(f"❌ Storage error when writing to: {path}: {e}", file=sys.stderr)
121
+ sys.exit(1)
122
+
123
+ def get_bytes(self, path: str) -> bytes:
124
+ absolute_url = self._get_absolute_url(path)
125
+ try:
126
+ obj = self.client.get_object(
127
+ Bucket=self.bucket,
128
+ Key=self._key(path)
129
+ )
130
+ print(f"📥 Data retrieved from: {absolute_url}")
131
+ return obj["Body"].read()
132
+ except EndpointConnectionError:
133
+ print(f"❌ Cannot connect to storage at {self.endpoint}", file=sys.stderr)
134
+ sys.exit(1)
135
+ except ClientError as e:
136
+ error_code = e.response['Error']['Code']
137
+ if error_code == 'NoSuchKey':
138
+ print(f"❌ File not found: {path}", file=sys.stderr)
139
+ elif error_code == 'AccessDenied':
140
+ print(f"❌ Access denied when reading: {path}", file=sys.stderr)
141
+ else:
142
+ print(f"❌ Storage error when reading: {path}: {e}", file=sys.stderr)
143
+ sys.exit(1)
144
+
145
+ def list(self, prefix: str = "") -> List[str]:
146
+ resp = self.client.list_objects_v2(
147
+ Bucket=self.bucket,
148
+ Prefix=self._key(prefix)
149
+ )
150
+
151
+ return [
152
+ obj["Key"]
153
+ for obj in resp.get("Contents", [])
154
+ ]
155
+
156
+
157
+ def get_storage(tier: str = "bronze") -> Storage:
158
+ return S3Storage(tier=tier)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -17,12 +17,19 @@ Classifier: Programming Language :: Python :: 3.11
17
17
  Classifier: Programming Language :: Python :: 3.12
18
18
  Requires-Python: >=3.8
19
19
  Description-Content-Type: text/markdown
20
+ Requires-Dist: boto3>=1.37.38
20
21
  Requires-Dist: click>=8.0.0
21
22
  Requires-Dist: pyyaml>=6.0
22
23
  Provides-Extra: dev
23
24
  Requires-Dist: pytest>=7.0; extra == "dev"
24
25
  Requires-Dist: black>=22.0; extra == "dev"
25
26
  Requires-Dist: isort>=5.0; extra == "dev"
27
+ Provides-Extra: polars
28
+ Requires-Dist: polars>=1.8.2; extra == "polars"
29
+ Requires-Dist: pyarrow>=17.0.0; extra == "polars"
30
+ Provides-Extra: pandas
31
+ Requires-Dist: pandas>=2.0.3; extra == "pandas"
32
+ Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
26
33
 
27
34
  # Steve CLI
28
35
 
@@ -40,21 +47,28 @@ uv pip install -e .
40
47
 
41
48
  ### Install from GitHub
42
49
 
50
+ `uv add steve-cli`
51
+
52
+ OR
53
+
43
54
  ```bash
44
55
  uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
45
56
  ```
46
57
 
58
+
59
+
47
60
  ## Usage
48
61
 
49
62
  ### Run a job
50
63
 
51
64
  ```bash
52
65
  steve extract-data
53
- steve transform-data
66
+ steve transform-data
54
67
  steve validate-data
55
68
  ```
56
69
 
57
70
  This will:
71
+
58
72
  1. Look for `jobs.yaml` in the current directory
59
73
  2. Find the specified job by name
60
74
  3. Set all environment variables from the job's `env` section
@@ -94,7 +108,7 @@ jobs:
94
108
  env:
95
109
  SOURCE_URL: "https://jsonplaceholder.typicode.com/users"
96
110
  OUTPUT_PATH: "/data/raw/extracted_data.parquet"
97
-
111
+
98
112
  - name: transform-data
99
113
  dependsOn: ["extract-data"]
100
114
  command: ["uv", "run", "src/transform.py"]
@@ -104,6 +118,7 @@ jobs:
104
118
  ```
105
119
 
106
120
  Running `steve extract-data` will:
121
+
107
122
  1. Set `SOURCE_URL=https://jsonplaceholder.typicode.com/users`
108
123
  2. Set `OUTPUT_PATH=/data/raw/extracted_data.parquet`
109
124
  3. Execute: `uv run src/extract.py`
@@ -141,3 +156,8 @@ pytest
141
156
  black steve_cli/
142
157
  isort steve_cli/
143
158
  ```
159
+
160
+ #
161
+
162
+ uv build
163
+ uv publish
@@ -2,6 +2,7 @@ README.md
2
2
  pyproject.toml
3
3
  steve_cli/__init__.py
4
4
  steve_cli/cli.py
5
+ steve_cli/storage.py
5
6
  steve_cli.egg-info/PKG-INFO
6
7
  steve_cli.egg-info/SOURCES.txt
7
8
  steve_cli.egg-info/dependency_links.txt
@@ -0,0 +1,16 @@
1
+ boto3>=1.37.38
2
+ click>=8.0.0
3
+ pyyaml>=6.0
4
+
5
+ [dev]
6
+ pytest>=7.0
7
+ black>=22.0
8
+ isort>=5.0
9
+
10
+ [pandas]
11
+ pandas>=2.0.3
12
+ pyarrow>=17.0.0
13
+
14
+ [polars]
15
+ polars>=1.8.2
16
+ pyarrow>=17.0.0
@@ -1,7 +0,0 @@
1
- click>=8.0.0
2
- pyyaml>=6.0
3
-
4
- [dev]
5
- pytest>=7.0
6
- black>=22.0
7
- isort>=5.0
File without changes