lam-cli 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lam/lam.py +132 -93
- lam_cli-0.1.3.dist-info/METADATA +52 -0
- lam_cli-0.1.3.dist-info/RECORD +8 -0
- lam_cli-0.1.2.dist-info/METADATA +0 -240
- lam_cli-0.1.2.dist-info/RECORD +0 -8
- {lam_cli-0.1.2.dist-info → lam_cli-0.1.3.dist-info}/LICENSE +0 -0
- {lam_cli-0.1.2.dist-info → lam_cli-0.1.3.dist-info}/WHEEL +0 -0
- {lam_cli-0.1.2.dist-info → lam_cli-0.1.3.dist-info}/entry_points.txt +0 -0
- {lam_cli-0.1.2.dist-info → lam_cli-0.1.3.dist-info}/top_level.txt +0 -0
lam/lam.py
CHANGED
|
@@ -22,9 +22,10 @@ from posthog import Posthog
|
|
|
22
22
|
posthog = Posthog(project_api_key='phc_wfeHFG0p5yZIdBpjVYy00o5x1HbEpggdMzIuFYgNPSK',
|
|
23
23
|
host='https://app.posthog.com')
|
|
24
24
|
|
|
25
|
-
logging.basicConfig(level=logging.
|
|
25
|
+
logging.basicConfig(level=logging.DEBUG,
|
|
26
26
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
27
27
|
logger = logging.getLogger(__name__)
|
|
28
|
+
logging.getLogger('urllib3').setLevel(logging.WARNING) # Suppress urllib3 logs
|
|
28
29
|
|
|
29
30
|
handler = LogtailHandler(source_token="TYz3WrrvC8ehYjXdAEGGyiDp")
|
|
30
31
|
logger.addHandler(handler)
|
|
@@ -47,20 +48,20 @@ class ResourceLimitError(LAMError):
|
|
|
47
48
|
|
|
48
49
|
def check_resource_limits(modules_dir: Optional[Path] = None) -> None:
|
|
49
50
|
"""Check system resource availability"""
|
|
50
|
-
|
|
51
|
+
logger.debug("Checking system resource limits")
|
|
51
52
|
disk = shutil.disk_usage(tempfile.gettempdir())
|
|
52
53
|
if disk.free < 100 * 1024 * 1024: # 100MB minimum
|
|
54
|
+
logger.critical("Insufficient disk space: %dMB free", disk.free // (1024*1024))
|
|
53
55
|
raise ResourceLimitError("Insufficient disk space")
|
|
54
56
|
|
|
55
|
-
# Check shared modules size if provided
|
|
56
57
|
if modules_dir and modules_dir.exists():
|
|
57
58
|
modules_size = sum(
|
|
58
59
|
os.path.getsize(os.path.join(dirpath, filename))
|
|
59
60
|
for dirpath, _, filenames in os.walk(modules_dir)
|
|
60
61
|
for filename in filenames
|
|
61
62
|
)
|
|
62
|
-
if modules_size > 500 * 1024 * 1024:
|
|
63
|
-
logger.warning("
|
|
63
|
+
if modules_size > 500 * 1024 * 1024:
|
|
64
|
+
logger.warning("Cleaning oversized modules directory (%dMB)", modules_size//(1024*1024))
|
|
64
65
|
shutil.rmtree(modules_dir)
|
|
65
66
|
modules_dir.mkdir(exist_ok=True)
|
|
66
67
|
|
|
@@ -71,7 +72,6 @@ class Stats:
|
|
|
71
72
|
self.memory_start = self.get_memory_usage()
|
|
72
73
|
|
|
73
74
|
def get_memory_usage(self):
|
|
74
|
-
import psutil
|
|
75
75
|
process = psutil.Process()
|
|
76
76
|
return process.memory_info().rss
|
|
77
77
|
|
|
@@ -120,17 +120,17 @@ class Engine:
|
|
|
120
120
|
class JQEngine(Engine):
|
|
121
121
|
"""JQ execution engine"""
|
|
122
122
|
def validate_environment(self) -> bool:
|
|
123
|
+
logger.debug("Validating JQ environment")
|
|
123
124
|
return shutil.which("jq") is not None
|
|
124
125
|
|
|
125
126
|
def execute(self, program_file: str, input_data: str) -> Tuple[Union[Dict, str], Optional[str]]:
|
|
126
127
|
logger.info(f"Executing JQ script: {program_file}")
|
|
127
128
|
|
|
128
129
|
try:
|
|
129
|
-
# Parse JQ program
|
|
130
130
|
with open(program_file, 'r') as file:
|
|
131
131
|
jq_script = ''.join(line for line in file if not line.strip().startswith('#'))
|
|
132
|
+
logger.debug("Loaded JQ script: %d characters", len(jq_script))
|
|
132
133
|
|
|
133
|
-
# Run JQ
|
|
134
134
|
process = subprocess.Popen(
|
|
135
135
|
["jq", "-c", jq_script],
|
|
136
136
|
stdin=subprocess.PIPE,
|
|
@@ -139,59 +139,82 @@ class JQEngine(Engine):
|
|
|
139
139
|
text=True
|
|
140
140
|
)
|
|
141
141
|
|
|
142
|
+
logger.debug("Starting JQ process PID %d", process.pid)
|
|
142
143
|
output, error = process.communicate(input=input_data)
|
|
143
144
|
|
|
144
145
|
if error:
|
|
146
|
+
logger.error("JQ error output: %s", error.strip())
|
|
145
147
|
raise ProcessingError(error)
|
|
146
148
|
|
|
147
149
|
# Handle output
|
|
148
150
|
try:
|
|
149
|
-
|
|
150
|
-
|
|
151
|
+
output_lines = [line.strip() for line in output.splitlines() if line.strip()]
|
|
152
|
+
logger.debug(f"Found {len(output_lines)} JSON objects in output")
|
|
153
|
+
|
|
154
|
+
if len(output_lines) > 1:
|
|
155
|
+
parsed = [json.loads(line) for line in output_lines]
|
|
156
|
+
logger.info(f"Processed {len(parsed)} JSON objects")
|
|
157
|
+
return {"lam.result": parsed}, None
|
|
158
|
+
elif len(output_lines) == 1:
|
|
159
|
+
result = json.loads(output_lines[0])
|
|
160
|
+
logger.info("Processed single JSON object")
|
|
161
|
+
return result, None
|
|
162
|
+
else:
|
|
163
|
+
logger.info("No JSON objects in output")
|
|
164
|
+
return {"lam.error": "No JSON objects in output"}, "No JSON objects in output"
|
|
165
|
+
|
|
166
|
+
except json.JSONDecodeError as e:
|
|
151
167
|
return {"lam.result": output}, None
|
|
152
168
|
|
|
153
169
|
except Exception as e:
|
|
170
|
+
logger.exception("JQ execution failed")
|
|
154
171
|
self.track_event('lam.jq.error', {'error': str(e)})
|
|
155
172
|
return {"lam.error": str(e)}, str(e)
|
|
156
173
|
|
|
157
174
|
class BunEngine(Engine):
|
|
158
|
-
"""Bun JavaScript execution engine"""
|
|
175
|
+
"""Bun JavaScript execution engine with enhanced logging"""
|
|
159
176
|
def __init__(self, *args, **kwargs):
|
|
160
177
|
super().__init__(*args, **kwargs)
|
|
161
|
-
# Create a persistent temp directory for node_modules
|
|
162
178
|
self.modules_dir = Path(tempfile.gettempdir()) / "lam_modules"
|
|
163
179
|
self.modules_dir.mkdir(exist_ok=True)
|
|
164
180
|
self._setup_shared_modules()
|
|
165
181
|
|
|
166
182
|
self.runtime_template = '''
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
183
|
+
const logs = [];
|
|
184
|
+
const originalLog = console.log;
|
|
185
|
+
const originalError = console.error;
|
|
186
|
+
const originalWarn = console.warn;
|
|
171
187
|
|
|
172
|
-
|
|
173
|
-
|
|
188
|
+
console.log = (...args) => logs.push({ type: 'log', message: args.map(String).join(' ') });
|
|
189
|
+
console.error = (...args) => {
|
|
190
|
+
originalError(...args); // Keep error output for debugging
|
|
191
|
+
logs.push({ type: 'error', message: args.map(String).join(' ') });
|
|
192
|
+
};
|
|
193
|
+
console.warn = (...args) => logs.push({ type: 'warn', message: args.map(String).join(' ') });
|
|
174
194
|
|
|
175
|
-
//
|
|
176
|
-
const
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
warn: console.warn
|
|
195
|
+
// Keep original stdout for result output
|
|
196
|
+
const writeResult = (obj) => {
|
|
197
|
+
console.error("Writing result:", JSON.stringify(obj, null, 2));
|
|
198
|
+
originalLog(JSON.stringify(obj));
|
|
180
199
|
};
|
|
181
|
-
globalThis.console = secureConsole;
|
|
182
200
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
201
|
+
const _ = require('lodash');
|
|
202
|
+
const { format, parseISO } = require('date-fns');
|
|
203
|
+
|
|
204
|
+
module.exports = {
|
|
205
|
+
_,
|
|
206
|
+
format,
|
|
207
|
+
parseISO,
|
|
208
|
+
logs,
|
|
209
|
+
writeResult
|
|
210
|
+
};
|
|
187
211
|
'''
|
|
188
|
-
|
|
212
|
+
|
|
189
213
|
def _setup_shared_modules(self):
|
|
190
214
|
"""Setup shared node_modules once"""
|
|
191
215
|
if not (self.modules_dir / "node_modules").exists():
|
|
192
|
-
|
|
216
|
+
logger.info("Initializing shared modules directory")
|
|
193
217
|
package_json = {
|
|
194
|
-
"type": "module",
|
|
195
218
|
"dependencies": {
|
|
196
219
|
"lodash": "^4.17.21",
|
|
197
220
|
"date-fns": "^2.30.0"
|
|
@@ -200,26 +223,25 @@ class BunEngine(Engine):
|
|
|
200
223
|
with open(self.modules_dir / "package.json", "w") as f:
|
|
201
224
|
json.dump(package_json, f, indent=2)
|
|
202
225
|
|
|
203
|
-
# Install dependencies once
|
|
204
226
|
try:
|
|
205
|
-
|
|
227
|
+
logger.debug("Installing shared dependencies")
|
|
228
|
+
result = subprocess.run(
|
|
206
229
|
[self.get_bun_path(), "install"],
|
|
207
230
|
cwd=self.modules_dir,
|
|
208
231
|
check=True,
|
|
209
232
|
capture_output=True,
|
|
210
233
|
text=True,
|
|
211
|
-
timeout=30
|
|
234
|
+
timeout=30
|
|
212
235
|
)
|
|
236
|
+
logger.debug("Dependency install output: %s", result.stdout)
|
|
213
237
|
except subprocess.CalledProcessError as e:
|
|
214
|
-
logger.error(
|
|
215
|
-
raise ProcessingError(
|
|
216
|
-
f"Failed to set up JavaScript environment: {e.stderr}"
|
|
217
|
-
) from e
|
|
238
|
+
logger.error("Dependency install failed: %s", e.stderr)
|
|
239
|
+
raise ProcessingError(f"Environment setup failed: {e.stderr}") from e
|
|
218
240
|
|
|
219
241
|
def create_wrapper(self, input_data: str, user_script: str) -> str:
|
|
220
242
|
"""Create the wrapper script with proper escaping"""
|
|
221
243
|
return f'''
|
|
222
|
-
|
|
244
|
+
const {{ _, format, parseISO, logs, writeResult }} = require('./runtime.js');
|
|
223
245
|
|
|
224
246
|
// Utility function to handle circular references in JSON.stringify
|
|
225
247
|
function safeStringify(obj) {{
|
|
@@ -250,7 +272,7 @@ class BunEngine(Engine):
|
|
|
250
272
|
// Parse input safely
|
|
251
273
|
let input;
|
|
252
274
|
try {{
|
|
253
|
-
input = {input_data};
|
|
275
|
+
input = JSON.parse({json.dumps(input_data)});
|
|
254
276
|
}} catch (e) {{
|
|
255
277
|
throw new Error(`Failed to parse input data: ${{e.message}}`);
|
|
256
278
|
}}
|
|
@@ -269,17 +291,15 @@ class BunEngine(Engine):
|
|
|
269
291
|
// Execute transform
|
|
270
292
|
const result = transform(input);
|
|
271
293
|
|
|
272
|
-
//
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
}}
|
|
277
|
-
throw new Error(`Result is not JSON serializable: ${{e.message}}`);
|
|
278
|
-
}}
|
|
294
|
+
// Output result after transform
|
|
295
|
+
writeResult({{
|
|
296
|
+
result,
|
|
297
|
+
logs
|
|
298
|
+
}});
|
|
279
299
|
}} catch (error) {{
|
|
280
300
|
console.error(JSON.stringify({{
|
|
281
301
|
error: error.message,
|
|
282
|
-
stack: error.stack,
|
|
302
|
+
stack: error.stack?.split('\\n') || [],
|
|
283
303
|
type: error.constructor.name
|
|
284
304
|
}}));
|
|
285
305
|
process.exit(1);
|
|
@@ -292,10 +312,11 @@ class BunEngine(Engine):
|
|
|
292
312
|
runtime_path = temp_dir / "runtime.js"
|
|
293
313
|
with open(runtime_path, "w") as f:
|
|
294
314
|
f.write(self.runtime_template)
|
|
315
|
+
logger.debug("Runtime file written to: %s", runtime_path)
|
|
295
316
|
|
|
296
317
|
# Symlink node_modules from shared directory
|
|
297
318
|
os.symlink(self.modules_dir / "node_modules", temp_dir / "node_modules")
|
|
298
|
-
|
|
319
|
+
logger.debug("node_modules symlinked from: %s", self.modules_dir / "node_modules")
|
|
299
320
|
|
|
300
321
|
def validate_environment(self) -> bool:
|
|
301
322
|
# Check multiple locations for bun
|
|
@@ -335,22 +356,23 @@ class BunEngine(Engine):
|
|
|
335
356
|
# Read user script
|
|
336
357
|
with open(program_file, 'r') as f:
|
|
337
358
|
user_script = f.read()
|
|
359
|
+
logger.debug("Loaded user script: %d characters", len(user_script))
|
|
338
360
|
|
|
339
361
|
# Create wrapper script
|
|
340
362
|
wrapper = self.create_wrapper(input_data, user_script)
|
|
341
|
-
|
|
342
363
|
script_path = temp_dir / "script.js"
|
|
343
|
-
with open(script_path,
|
|
364
|
+
with open(script_path, 'w') as f:
|
|
344
365
|
f.write(wrapper)
|
|
366
|
+
logger.debug("Generated wrapper script: %s", script_path)
|
|
345
367
|
|
|
346
368
|
# Execute with Bun
|
|
347
369
|
process = subprocess.Popen(
|
|
348
370
|
[
|
|
349
371
|
self.get_bun_path(),
|
|
350
372
|
"run",
|
|
351
|
-
"--no-fetch",
|
|
352
|
-
"--smol",
|
|
353
|
-
"--silent",
|
|
373
|
+
"--no-fetch",
|
|
374
|
+
"--smol",
|
|
375
|
+
"--silent",
|
|
354
376
|
str(script_path)
|
|
355
377
|
],
|
|
356
378
|
stdout=subprocess.PIPE,
|
|
@@ -358,36 +380,64 @@ class BunEngine(Engine):
|
|
|
358
380
|
text=True,
|
|
359
381
|
cwd=temp_dir
|
|
360
382
|
)
|
|
383
|
+
logger.info("Started Bun process PID %d", process.pid)
|
|
361
384
|
|
|
362
385
|
try:
|
|
363
|
-
output, error = process.communicate(timeout=5)
|
|
386
|
+
output, error = process.communicate(timeout=5)
|
|
387
|
+
logger.debug("Process completed with code %d", process.returncode)
|
|
364
388
|
except subprocess.TimeoutExpired as e:
|
|
389
|
+
logger.warning("Process timeout after 5 seconds")
|
|
365
390
|
process.kill()
|
|
366
|
-
|
|
391
|
+
return {"lam.error": "Script execution timed out"}, "Execution timed out after 5 seconds"
|
|
367
392
|
|
|
368
|
-
|
|
393
|
+
# Handle process errors
|
|
394
|
+
if process.returncode != 0:
|
|
369
395
|
try:
|
|
370
|
-
|
|
396
|
+
# Try to parse structured error from stderr
|
|
397
|
+
error_data = json.loads(error.strip())
|
|
371
398
|
error_msg = error_data.get('error', 'Unknown error')
|
|
372
|
-
|
|
373
|
-
|
|
399
|
+
stack = error_data.get('stack', [])
|
|
400
|
+
|
|
401
|
+
# Format error message
|
|
402
|
+
error_details = {
|
|
403
|
+
"lam.error": error_msg,
|
|
404
|
+
"stack_trace": stack
|
|
405
|
+
}
|
|
406
|
+
return error_details, error_msg
|
|
407
|
+
|
|
374
408
|
except json.JSONDecodeError:
|
|
375
|
-
|
|
376
|
-
|
|
409
|
+
# Fallback to raw error output
|
|
410
|
+
error_msg = error.strip() or "Unknown error"
|
|
411
|
+
return {"lam.error": error_msg}, error_msg
|
|
377
412
|
|
|
413
|
+
# Handle successful output
|
|
378
414
|
try:
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
415
|
+
output_data = json.loads(output)
|
|
416
|
+
|
|
417
|
+
# Process JavaScript logs (if any)
|
|
418
|
+
if 'logs' in output_data:
|
|
419
|
+
for log_entry in output_data.get('logs', []):
|
|
420
|
+
if log_entry['type'] == 'error':
|
|
421
|
+
logger.error("[JS] %s", log_entry['message'])
|
|
422
|
+
else:
|
|
423
|
+
logger.debug("[JS] %s", log_entry['message'])
|
|
424
|
+
|
|
425
|
+
result = output_data.get('result', {})
|
|
426
|
+
return result, None
|
|
427
|
+
|
|
428
|
+
except json.JSONDecodeError as e:
|
|
429
|
+
logger.error("Failed to parse output: %s", str(e))
|
|
430
|
+
return {
|
|
431
|
+
"lam.error": "Invalid JSON output",
|
|
432
|
+
"raw_output": output.strip()
|
|
433
|
+
}, "Output format error"
|
|
382
434
|
|
|
383
435
|
except Exception as e:
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
})
|
|
390
|
-
return {"lam.error": str(e)}, str(e)
|
|
436
|
+
logger.exception("Execution failed")
|
|
437
|
+
return {
|
|
438
|
+
"lam.error": str(e),
|
|
439
|
+
"type": e.__class__.__name__
|
|
440
|
+
}, str(e)
|
|
391
441
|
|
|
392
442
|
def get_engine(engine_type: str, workspace_id: str, flow_id: str, execution_id: str) -> Engine:
|
|
393
443
|
"""Factory function to get the appropriate execution engine"""
|
|
@@ -409,13 +459,16 @@ def get_engine(engine_type: str, workspace_id: str, flow_id: str, execution_id:
|
|
|
409
459
|
def process_input(input: str) -> Tuple[str, Optional[str]]:
|
|
410
460
|
"""Process and validate input data"""
|
|
411
461
|
if os.path.isfile(input):
|
|
462
|
+
logger.debug("Loading input from file: %s", input)
|
|
412
463
|
with open(input, 'r') as file:
|
|
413
464
|
return file.read(), None
|
|
414
465
|
|
|
415
466
|
try:
|
|
416
467
|
json.loads(input)
|
|
468
|
+
logger.debug("Validated inline JSON input")
|
|
417
469
|
return input, None
|
|
418
470
|
except json.JSONDecodeError as e:
|
|
471
|
+
logger.error("Invalid JSON input: %s", str(e))
|
|
419
472
|
return None, str(e)
|
|
420
473
|
|
|
421
474
|
@click.group()
|
|
@@ -435,16 +488,14 @@ def lam():
|
|
|
435
488
|
def run(program_file: str, input: str, language: str, workspace_id: str,
|
|
436
489
|
flow_id: str, execution_id: str, as_json: bool):
|
|
437
490
|
"""Execute a LAM transformation script"""
|
|
438
|
-
stats = Stats()
|
|
491
|
+
stats = Stats()
|
|
439
492
|
|
|
440
|
-
# Initialize engine
|
|
441
493
|
try:
|
|
442
494
|
engine = get_engine(language, workspace_id, flow_id, execution_id)
|
|
443
495
|
except (ValueError, EnvironmentError) as e:
|
|
444
496
|
click.echo({"lam.error": str(e)}, err=True)
|
|
445
497
|
return
|
|
446
498
|
|
|
447
|
-
# Setup logging
|
|
448
499
|
log_file = engine.get_log_file()
|
|
449
500
|
result_file = engine.get_result_file()
|
|
450
501
|
|
|
@@ -452,25 +503,22 @@ def run(program_file: str, input: str, language: str, workspace_id: str,
|
|
|
452
503
|
file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
|
|
453
504
|
logger.addHandler(file_handler)
|
|
454
505
|
|
|
455
|
-
logger.info(
|
|
506
|
+
logger.info("Starting LAM execution with %s engine", language)
|
|
456
507
|
engine.track_event('lam.run.start', {
|
|
457
508
|
'language': language,
|
|
458
509
|
'program_file': program_file
|
|
459
510
|
})
|
|
460
511
|
|
|
461
512
|
try:
|
|
462
|
-
# Process input
|
|
463
513
|
input_data, error = process_input(input)
|
|
464
514
|
if error:
|
|
465
515
|
raise ProcessingError(f"Invalid input: {error}")
|
|
466
516
|
|
|
467
|
-
# Execute transformation
|
|
468
517
|
result, error = engine.execute(program_file, input_data)
|
|
469
518
|
|
|
470
|
-
# Get final stats
|
|
471
519
|
stats_data = stats.finalize()
|
|
472
|
-
logger.info(
|
|
473
|
-
|
|
520
|
+
logger.info("Execution stats: duration=%.2fms, memory=%.2fMB",
|
|
521
|
+
stats_data['duration_ms'], stats_data['memory_used_mb'])
|
|
474
522
|
|
|
475
523
|
if error:
|
|
476
524
|
click.echo({"lam.error": error}, err=True)
|
|
@@ -480,15 +528,6 @@ def run(program_file: str, input: str, language: str, workspace_id: str,
|
|
|
480
528
|
click.echo(output)
|
|
481
529
|
engine.track_event('lam.run.success', stats_data)
|
|
482
530
|
|
|
483
|
-
# Save result with stats
|
|
484
|
-
result_with_stats = {
|
|
485
|
-
'result': result,
|
|
486
|
-
'stats': stats_data,
|
|
487
|
-
'error': error or None
|
|
488
|
-
}
|
|
489
|
-
logger.info(f"Final stats: {stats_data}")
|
|
490
|
-
|
|
491
|
-
# If result is an array, wrap it in "lam.result"
|
|
492
531
|
if isinstance(result, list):
|
|
493
532
|
result = {"lam.result": result}
|
|
494
533
|
|
|
@@ -497,9 +536,9 @@ def run(program_file: str, input: str, language: str, workspace_id: str,
|
|
|
497
536
|
|
|
498
537
|
except Exception as e:
|
|
499
538
|
stats_data = stats.finalize()
|
|
500
|
-
logger.error(
|
|
501
|
-
logger.error(
|
|
502
|
-
|
|
539
|
+
logger.error("Execution failed: %s", str(e))
|
|
540
|
+
logger.error("Final stats: duration=%.2fms, memory=%.2fMB",
|
|
541
|
+
stats_data['duration_ms'], stats_data['memory_used_mb'])
|
|
503
542
|
click.echo({"lam.error": str(e)}, err=True)
|
|
504
543
|
engine.track_event('lam.run.error', {'error': str(e), **stats_data})
|
|
505
544
|
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: lam-cli
|
|
3
|
+
Version: 0.1.3
|
|
4
|
+
Summary: Secure data transformation tool supporting JQ and JavaScript (Bun)
|
|
5
|
+
Home-page: https://github.com/laminar-run/lam
|
|
6
|
+
Author: Laminar Run, Inc.
|
|
7
|
+
Author-email: connect@laminar.run
|
|
8
|
+
License: GPLv3
|
|
9
|
+
Project-URL: Documentation, https://docs.laminar.run
|
|
10
|
+
Project-URL: Source, https://github.com/laminar-run/lam
|
|
11
|
+
Project-URL: Issue Tracker, https://github.com/laminar-run/lam/issues
|
|
12
|
+
Keywords: laminar,api,integration,transformation,json,jq,javascript,bun
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
24
|
+
Classifier: Topic :: Software Development :: Build Tools
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: backoff>=2.2.1
|
|
29
|
+
Requires-Dist: certifi>=2024.12.14
|
|
30
|
+
Requires-Dist: charset-normalizer>=3.3.2
|
|
31
|
+
Requires-Dist: click>=8.1.7
|
|
32
|
+
Requires-Dist: idna>=3.7
|
|
33
|
+
Requires-Dist: logtail-python>=0.2.2
|
|
34
|
+
Requires-Dist: monotonic>=1.6
|
|
35
|
+
Requires-Dist: msgpack>=1.0.8
|
|
36
|
+
Requires-Dist: posthog>=3.4.0
|
|
37
|
+
Requires-Dist: psutil>=5.9.0
|
|
38
|
+
Requires-Dist: python-dateutil>=2.8.2
|
|
39
|
+
Requires-Dist: requests>=2.32.3
|
|
40
|
+
Requires-Dist: six>=1.16.0
|
|
41
|
+
Requires-Dist: urllib3>=2.2.2
|
|
42
|
+
Dynamic: author
|
|
43
|
+
Dynamic: author-email
|
|
44
|
+
Dynamic: classifier
|
|
45
|
+
Dynamic: description-content-type
|
|
46
|
+
Dynamic: home-page
|
|
47
|
+
Dynamic: keywords
|
|
48
|
+
Dynamic: license
|
|
49
|
+
Dynamic: project-url
|
|
50
|
+
Dynamic: requires-dist
|
|
51
|
+
Dynamic: requires-python
|
|
52
|
+
Dynamic: summary
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
lam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
lam/lam.py,sha256=MrJKA_tFVEEvnhKJkMtrDbdL50bX27aJ0WUbP2dUJiU,20981
|
|
3
|
+
lam_cli-0.1.3.dist-info/LICENSE,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
|
|
4
|
+
lam_cli-0.1.3.dist-info/METADATA,sha256=urxYh-mNjUh3v-ggzB_gDIdp5JOa1aQPVKx9kPQGSio,1895
|
|
5
|
+
lam_cli-0.1.3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
6
|
+
lam_cli-0.1.3.dist-info/entry_points.txt,sha256=iJSsJitcGMikKJ3Q9KNWxEy911oTrSxGSO1HiwcBrKE,40
|
|
7
|
+
lam_cli-0.1.3.dist-info/top_level.txt,sha256=WyM7-Ig60qQH9meqS293pEd83jrMtbvGJM8ALZOQCtA,4
|
|
8
|
+
lam_cli-0.1.3.dist-info/RECORD,,
|
lam_cli-0.1.2.dist-info/METADATA
DELETED
|
@@ -1,240 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.2
|
|
2
|
-
Name: lam-cli
|
|
3
|
-
Version: 0.1.2
|
|
4
|
-
Summary: Secure data transformation tool supporting JQ and JavaScript (Bun)
|
|
5
|
-
Home-page: https://github.com/laminar-run/lam
|
|
6
|
-
Author: Laminar Run, Inc.
|
|
7
|
-
Author-email: connect@laminar.run
|
|
8
|
-
License: GPLv3
|
|
9
|
-
Project-URL: Documentation, https://docs.laminar.run
|
|
10
|
-
Project-URL: Source, https://github.com/laminar-run/lam
|
|
11
|
-
Project-URL: Issue Tracker, https://github.com/laminar-run/lam/issues
|
|
12
|
-
Keywords: laminar,api,integration,transformation,json,jq,javascript,bun
|
|
13
|
-
Classifier: Development Status :: 4 - Beta
|
|
14
|
-
Classifier: Environment :: Console
|
|
15
|
-
Classifier: Intended Audience :: Developers
|
|
16
|
-
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
17
|
-
Classifier: Operating System :: OS Independent
|
|
18
|
-
Classifier: Programming Language :: Python :: 3
|
|
19
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
-
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
-
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
24
|
-
Classifier: Topic :: Software Development :: Build Tools
|
|
25
|
-
Requires-Python: >=3.9
|
|
26
|
-
Description-Content-Type: text/markdown
|
|
27
|
-
License-File: LICENSE
|
|
28
|
-
Requires-Dist: backoff>=2.2.1
|
|
29
|
-
Requires-Dist: certifi>=2024.12.14
|
|
30
|
-
Requires-Dist: charset-normalizer>=3.3.2
|
|
31
|
-
Requires-Dist: click>=8.1.7
|
|
32
|
-
Requires-Dist: idna>=3.7
|
|
33
|
-
Requires-Dist: logtail-python>=0.2.2
|
|
34
|
-
Requires-Dist: monotonic>=1.6
|
|
35
|
-
Requires-Dist: msgpack>=1.0.8
|
|
36
|
-
Requires-Dist: posthog>=3.4.0
|
|
37
|
-
Requires-Dist: psutil>=5.9.0
|
|
38
|
-
Requires-Dist: python-dateutil>=2.8.2
|
|
39
|
-
Requires-Dist: requests>=2.32.3
|
|
40
|
-
Requires-Dist: six>=1.16.0
|
|
41
|
-
Requires-Dist: urllib3>=2.2.2
|
|
42
|
-
Dynamic: author
|
|
43
|
-
Dynamic: author-email
|
|
44
|
-
Dynamic: classifier
|
|
45
|
-
Dynamic: description
|
|
46
|
-
Dynamic: description-content-type
|
|
47
|
-
Dynamic: home-page
|
|
48
|
-
Dynamic: keywords
|
|
49
|
-
Dynamic: license
|
|
50
|
-
Dynamic: project-url
|
|
51
|
-
Dynamic: requires-dist
|
|
52
|
-
Dynamic: requires-python
|
|
53
|
-
Dynamic: summary
|
|
54
|
-
|
|
55
|
-
# lam
|
|
56
|
-
Lam is a data transformation tool for Laminar that supports both `jq` and JavaScript transformations using Bun.
|
|
57
|
-
|
|
58
|
-
## Quickstart
|
|
59
|
-
Install the dependencies:
|
|
60
|
-
```bash
|
|
61
|
-
# For JQ support
|
|
62
|
-
brew install jq # or sudo apt-get install jq
|
|
63
|
-
|
|
64
|
-
# For JavaScript support
|
|
65
|
-
curl -fsSL https://bun.sh/install | bash
|
|
66
|
-
|
|
67
|
-
make setup
|
|
68
|
-
```
|
|
69
|
-
|
|
70
|
-
Run the CLI tool:
|
|
71
|
-
```bash
|
|
72
|
-
make cli ARGS="run <program> <input> [--language jq|js]"
|
|
73
|
-
```
|
|
74
|
-
|
|
75
|
-
## Features
|
|
76
|
-
- JQ transformations (default)
|
|
77
|
-
- JavaScript transformations with Bun runtime
|
|
78
|
-
- Built-in utilities (lodash, date-fns)
|
|
79
|
-
- Resource monitoring and limits
|
|
80
|
-
- Detailed execution statistics
|
|
81
|
-
- Secure execution environment
|
|
82
|
-
|
|
83
|
-
## Examples
|
|
84
|
-
|
|
85
|
-
### JQ Transform
|
|
86
|
-
```bash
|
|
87
|
-
make cli ARGS="run examples/transform.jq data.json"
|
|
88
|
-
```
|
|
89
|
-
|
|
90
|
-
### JavaScript Transform
|
|
91
|
-
```bash
|
|
92
|
-
make cli ARGS="run examples/transform.js data.json --language js"
|
|
93
|
-
```
|
|
94
|
-
|
|
95
|
-
Example JavaScript transform:
|
|
96
|
-
```javascript
|
|
97
|
-
(input) => {
|
|
98
|
-
// Lodash available as _
|
|
99
|
-
return _.map(input.data, item => ({
|
|
100
|
-
value: item.value * 2
|
|
101
|
-
}));
|
|
102
|
-
}
|
|
103
|
-
```
|
|
104
|
-
|
|
105
|
-
## Installation
|
|
106
|
-
|
|
107
|
-
### Docker Installation
|
|
108
|
-
```dockerfile
|
|
109
|
-
# Install lam-cli
|
|
110
|
-
RUN pip3 install git+https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/user/project.git@{version}
|
|
111
|
-
|
|
112
|
-
# Install dependencies
|
|
113
|
-
RUN apt-get update && apt-get install -y jq
|
|
114
|
-
RUN curl -fsSL https://bun.sh/install | bash
|
|
115
|
-
```
|
|
116
|
-
|
|
117
|
-
### Manual Setup
|
|
118
|
-
Create a virtual environment and install dependencies:
|
|
119
|
-
```bash
|
|
120
|
-
python3 -m venv ./venv
|
|
121
|
-
source ./venv/bin/activate
|
|
122
|
-
pip install -r requirements.txt
|
|
123
|
-
```
|
|
124
|
-
|
|
125
|
-
## Usage
|
|
126
|
-
```bash
|
|
127
|
-
# Basic usage
|
|
128
|
-
python3 ./lam/lam.py run <program> <input>
|
|
129
|
-
|
|
130
|
-
# With JavaScript
|
|
131
|
-
python3 ./lam/lam.py run script.js data.json --language js
|
|
132
|
-
|
|
133
|
-
# Full options
|
|
134
|
-
python3 ./lam/lam.py run <program> <input> \
|
|
135
|
-
--language [jq|js] \
|
|
136
|
-
--workspace_id <id> \
|
|
137
|
-
--flow_id <id> \
|
|
138
|
-
--execution_id <id> \
|
|
139
|
-
[--as-json]
|
|
140
|
-
```
|
|
141
|
-
|
|
142
|
-
## Resource Limits
|
|
143
|
-
- Maximum input size: 10MB
|
|
144
|
-
- Execution timeout: 5 seconds
|
|
145
|
-
- Memory limits enabled
|
|
146
|
-
- Disk space monitoring
|
|
147
|
-
|
|
148
|
-
## Security
|
|
149
|
-
- Sandboxed JavaScript execution
|
|
150
|
-
- Network access disabled
|
|
151
|
-
- Limited global scope
|
|
152
|
-
- Resource monitoring
|
|
153
|
-
- Secure dependency management
|
|
154
|
-
|
|
155
|
-
## Logging and Monitoring
|
|
156
|
-
- Execution statistics (duration, memory usage)
|
|
157
|
-
- Detailed error tracking
|
|
158
|
-
- PostHog analytics integration
|
|
159
|
-
- Log file generation
|
|
160
|
-
|
|
161
|
-
## Development
|
|
162
|
-
```bash
|
|
163
|
-
# Run all tests
|
|
164
|
-
make test
|
|
165
|
-
|
|
166
|
-
# Run specific test suite
|
|
167
|
-
make test-jq
|
|
168
|
-
make test-js
|
|
169
|
-
make test-js-edge-cases
|
|
170
|
-
|
|
171
|
-
# Run single test
|
|
172
|
-
make test-single TEST=test/js/example.js DATA=test/data/input.json
|
|
173
|
-
```
|
|
174
|
-
|
|
175
|
-
## Releases
|
|
176
|
-
Update version in `setup.py`:
|
|
177
|
-
```python
|
|
178
|
-
setup(
|
|
179
|
-
name="lam-cli",
|
|
180
|
-
version="0.0.<x>",
|
|
181
|
-
...
|
|
182
|
-
)
|
|
183
|
-
```
|
|
184
|
-
|
|
185
|
-
Create and push tag:
|
|
186
|
-
```bash
|
|
187
|
-
git tag v<version>-<increment>
|
|
188
|
-
git push origin v<version>-<increment>
|
|
189
|
-
```
|
|
190
|
-
|
|
191
|
-
## Dependencies
|
|
192
|
-
Update dependencies:
|
|
193
|
-
```bash
|
|
194
|
-
pip3 install <package>
|
|
195
|
-
pip3 freeze > requirements.txt
|
|
196
|
-
```
|
|
197
|
-
|
|
198
|
-
## Install Locally
|
|
199
|
-
|
|
200
|
-
```bash
|
|
201
|
-
pip3 install -e .
|
|
202
|
-
```
|
|
203
|
-
|
|
204
|
-
## Troubleshooting
|
|
205
|
-
|
|
206
|
-
### Package Installation Issues
|
|
207
|
-
If you encounter issues installing the package with pip, particularly with the certifi dependency, you may see errors like:
|
|
208
|
-
|
|
209
|
-
```bash
|
|
210
|
-
error: uninstall-no-record-file
|
|
211
|
-
× Cannot uninstall certifi None
|
|
212
|
-
╰─> The package's contents are unknown: no RECORD file was found for certifi.
|
|
213
|
-
```
|
|
214
|
-
This can happen when multiple Python versions are installed or when system packages have been modified. Try these steps:
|
|
215
|
-
|
|
216
|
-
First, identify which Python version you're using:
|
|
217
|
-
|
|
218
|
-
```bash
|
|
219
|
-
which python3
|
|
220
|
-
python3 --version
|
|
221
|
-
which pip3
|
|
222
|
-
```
|
|
223
|
-
|
|
224
|
-
Remove the problematic certifi installation (adjust path based on your Python version):
|
|
225
|
-
|
|
226
|
-
```bash
|
|
227
|
-
sudo rm -rf /usr/local/lib/python3.13/site-packages/certifi
|
|
228
|
-
```
|
|
229
|
-
|
|
230
|
-
Install certifi directly:
|
|
231
|
-
|
|
232
|
-
```bash
|
|
233
|
-
pip3 install --ignore-installed certifi --break-system-packages
|
|
234
|
-
```
|
|
235
|
-
|
|
236
|
-
Then try installing lam-cli again:
|
|
237
|
-
|
|
238
|
-
```bash
|
|
239
|
-
pip3 install . --break-system-packages
|
|
240
|
-
```
|
lam_cli-0.1.2.dist-info/RECORD
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
lam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
lam/lam.py,sha256=S3DOSyLLmbpUPpDmPsM6ND-TQUZs3yIZd2Lwx6XirVs,18126
|
|
3
|
-
lam_cli-0.1.2.dist-info/LICENSE,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
|
|
4
|
-
lam_cli-0.1.2.dist-info/METADATA,sha256=fPWtIn4tKSom9fh77-jro4o-zWSXlEvo_K4OkF5cRYI,5645
|
|
5
|
-
lam_cli-0.1.2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
6
|
-
lam_cli-0.1.2.dist-info/entry_points.txt,sha256=iJSsJitcGMikKJ3Q9KNWxEy911oTrSxGSO1HiwcBrKE,40
|
|
7
|
-
lam_cli-0.1.2.dist-info/top_level.txt,sha256=WyM7-Ig60qQH9meqS293pEd83jrMtbvGJM8ALZOQCtA,4
|
|
8
|
-
lam_cli-0.1.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|