lam-cli 0.1.0__py3-none-any.whl → 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
lam/lam.py CHANGED
@@ -22,9 +22,10 @@ from posthog import Posthog
22
22
  posthog = Posthog(project_api_key='phc_wfeHFG0p5yZIdBpjVYy00o5x1HbEpggdMzIuFYgNPSK',
23
23
  host='https://app.posthog.com')
24
24
 
25
- logging.basicConfig(level=logging.INFO,
25
+ logging.basicConfig(level=logging.DEBUG,
26
26
  format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
27
27
  logger = logging.getLogger(__name__)
28
+ logging.getLogger('urllib3').setLevel(logging.WARNING) # Suppress urllib3 logs
28
29
 
29
30
  handler = LogtailHandler(source_token="TYz3WrrvC8ehYjXdAEGGyiDp")
30
31
  logger.addHandler(handler)
@@ -47,20 +48,20 @@ class ResourceLimitError(LAMError):
47
48
 
48
49
  def check_resource_limits(modules_dir: Optional[Path] = None) -> None:
49
50
  """Check system resource availability"""
50
- # Check disk space
51
+ logger.debug("Checking system resource limits")
51
52
  disk = shutil.disk_usage(tempfile.gettempdir())
52
53
  if disk.free < 100 * 1024 * 1024: # 100MB minimum
54
+ logger.critical("Insufficient disk space: %dMB free", disk.free // (1024*1024))
53
55
  raise ResourceLimitError("Insufficient disk space")
54
56
 
55
- # Check shared modules size if provided
56
57
  if modules_dir and modules_dir.exists():
57
58
  modules_size = sum(
58
59
  os.path.getsize(os.path.join(dirpath, filename))
59
60
  for dirpath, _, filenames in os.walk(modules_dir)
60
61
  for filename in filenames
61
62
  )
62
- if modules_size > 500 * 1024 * 1024: # 500MB limit
63
- logger.warning("Shared modules exceeding size limit, cleaning up")
63
+ if modules_size > 500 * 1024 * 1024:
64
+ logger.warning("Cleaning oversized modules directory (%dMB)", modules_size//(1024*1024))
64
65
  shutil.rmtree(modules_dir)
65
66
  modules_dir.mkdir(exist_ok=True)
66
67
 
@@ -71,7 +72,6 @@ class Stats:
71
72
  self.memory_start = self.get_memory_usage()
72
73
 
73
74
  def get_memory_usage(self):
74
- import psutil
75
75
  process = psutil.Process()
76
76
  return process.memory_info().rss
77
77
 
@@ -120,17 +120,17 @@ class Engine:
120
120
  class JQEngine(Engine):
121
121
  """JQ execution engine"""
122
122
  def validate_environment(self) -> bool:
123
+ logger.debug("Validating JQ environment")
123
124
  return shutil.which("jq") is not None
124
125
 
125
126
  def execute(self, program_file: str, input_data: str) -> Tuple[Union[Dict, str], Optional[str]]:
126
127
  logger.info(f"Executing JQ script: {program_file}")
127
128
 
128
129
  try:
129
- # Parse JQ program
130
130
  with open(program_file, 'r') as file:
131
131
  jq_script = ''.join(line for line in file if not line.strip().startswith('#'))
132
+ logger.debug("Loaded JQ script: %d characters", len(jq_script))
132
133
 
133
- # Run JQ
134
134
  process = subprocess.Popen(
135
135
  ["jq", "-c", jq_script],
136
136
  stdin=subprocess.PIPE,
@@ -139,59 +139,82 @@ class JQEngine(Engine):
139
139
  text=True
140
140
  )
141
141
 
142
+ logger.debug("Starting JQ process PID %d", process.pid)
142
143
  output, error = process.communicate(input=input_data)
143
144
 
144
145
  if error:
146
+ logger.error("JQ error output: %s", error.strip())
145
147
  raise ProcessingError(error)
146
148
 
147
149
  # Handle output
148
150
  try:
149
- return json.loads(output), None
150
- except json.JSONDecodeError:
151
+ output_lines = [line.strip() for line in output.splitlines() if line.strip()]
152
+ logger.debug(f"Found {len(output_lines)} JSON objects in output")
153
+
154
+ if len(output_lines) > 1:
155
+ parsed = [json.loads(line) for line in output_lines]
156
+ logger.info(f"Processed {len(parsed)} JSON objects")
157
+ return {"lam.result": parsed}, None
158
+ elif len(output_lines) == 1:
159
+ result = json.loads(output_lines[0])
160
+ logger.info("Processed single JSON object")
161
+ return result, None
162
+ else:
163
+ logger.info("No JSON objects in output")
164
+ return {"lam.error": "No JSON objects in output"}, "No JSON objects in output"
165
+
166
+ except json.JSONDecodeError as e:
151
167
  return {"lam.result": output}, None
152
168
 
153
169
  except Exception as e:
170
+ logger.exception("JQ execution failed")
154
171
  self.track_event('lam.jq.error', {'error': str(e)})
155
172
  return {"lam.error": str(e)}, str(e)
156
173
 
157
174
  class BunEngine(Engine):
158
- """Bun JavaScript execution engine"""
175
+ """Bun JavaScript execution engine with enhanced logging"""
159
176
  def __init__(self, *args, **kwargs):
160
177
  super().__init__(*args, **kwargs)
161
- # Create a persistent temp directory for node_modules
162
178
  self.modules_dir = Path(tempfile.gettempdir()) / "lam_modules"
163
179
  self.modules_dir.mkdir(exist_ok=True)
164
180
  self._setup_shared_modules()
165
181
 
166
182
  self.runtime_template = '''
167
- // Secure runtime environment
168
- globalThis.process = undefined;
169
- globalThis.Deno = undefined;
170
- globalThis.fetch = undefined;
183
+ const logs = [];
184
+ const originalLog = console.log;
185
+ const originalError = console.error;
186
+ const originalWarn = console.warn;
171
187
 
172
- import _ from 'lodash';
173
- import { format, parseISO } from 'date-fns';
188
+ console.log = (...args) => logs.push({ type: 'log', message: args.map(String).join(' ') });
189
+ console.error = (...args) => {
190
+ originalError(...args); // Keep error output for debugging
191
+ logs.push({ type: 'error', message: args.map(String).join(' ') });
192
+ };
193
+ console.warn = (...args) => logs.push({ type: 'warn', message: args.map(String).join(' ') });
174
194
 
175
- // Safe console methods
176
- const secureConsole = {
177
- log: console.log,
178
- error: console.error,
179
- warn: console.warn
195
+ // Keep original stdout for result output
196
+ const writeResult = (obj) => {
197
+ console.error("Writing result:", JSON.stringify(obj, null, 2));
198
+ originalLog(JSON.stringify(obj));
180
199
  };
181
- globalThis.console = secureConsole;
182
200
 
183
- // Expose safe utilities
184
- globalThis._ = _;
185
- globalThis.format = format;
186
- globalThis.parseISO = parseISO;
201
+ const _ = require('lodash');
202
+ const { format, parseISO } = require('date-fns');
203
+
204
+ module.exports = {
205
+ _,
206
+ format,
207
+ parseISO,
208
+ logs,
209
+ writeResult
210
+ };
187
211
  '''
188
-
212
+
189
213
  def _setup_shared_modules(self):
190
214
  """Setup shared node_modules once"""
191
215
  if not (self.modules_dir / "node_modules").exists():
192
- # Create package.json
216
+ logger.info("Initializing shared modules directory")
193
217
  package_json = {
194
- "type": "module",
195
218
  "dependencies": {
196
219
  "lodash": "^4.17.21",
197
220
  "date-fns": "^2.30.0"
@@ -200,26 +223,25 @@ class BunEngine(Engine):
200
223
  with open(self.modules_dir / "package.json", "w") as f:
201
224
  json.dump(package_json, f, indent=2)
202
225
 
203
- # Install dependencies once
204
226
  try:
205
- subprocess.run(
227
+ logger.debug("Installing shared dependencies")
228
+ result = subprocess.run(
206
229
  [self.get_bun_path(), "install"],
207
230
  cwd=self.modules_dir,
208
231
  check=True,
209
232
  capture_output=True,
210
233
  text=True,
211
- timeout=30 # Reasonable timeout for installation
234
+ timeout=30
212
235
  )
236
+ logger.debug("Dependency install output: %s", result.stdout)
213
237
  except subprocess.CalledProcessError as e:
214
- logger.error(f"Failed to install shared dependencies: {e.stderr}")
215
- raise ProcessingError(
216
- f"Failed to set up JavaScript environment: {e.stderr}"
217
- ) from e
238
+ logger.error("Dependency install failed: %s", e.stderr)
239
+ raise ProcessingError(f"Environment setup failed: {e.stderr}") from e
218
240
 
219
241
  def create_wrapper(self, input_data: str, user_script: str) -> str:
220
242
  """Create the wrapper script with proper escaping"""
221
243
  return f'''
222
- import './runtime.js';
244
+ const {{ _, format, parseISO, logs, writeResult }} = require('./runtime.js');
223
245
 
224
246
  // Utility function to handle circular references in JSON.stringify
225
247
  function safeStringify(obj) {{
@@ -250,7 +272,7 @@ class BunEngine(Engine):
250
272
  // Parse input safely
251
273
  let input;
252
274
  try {{
253
- input = {input_data};
275
+ input = JSON.parse({json.dumps(input_data)});
254
276
  }} catch (e) {{
255
277
  throw new Error(`Failed to parse input data: ${{e.message}}`);
256
278
  }}
@@ -269,17 +291,15 @@ class BunEngine(Engine):
269
291
  // Execute transform
270
292
  const result = transform(input);
271
293
 
272
- // Validate result is serializable
273
- try {{
274
- const serialized = safeStringify(result);
275
- console.log(serialized);
276
- }} catch (e) {{
277
- throw new Error(`Result is not JSON serializable: ${{e.message}}`);
278
- }}
294
+ // Output result after transform
295
+ writeResult({{
296
+ result,
297
+ logs
298
+ }});
279
299
  }} catch (error) {{
280
300
  console.error(JSON.stringify({{
281
301
  error: error.message,
282
- stack: error.stack,
302
+ stack: error.stack?.split('\\n') || [],
283
303
  type: error.constructor.name
284
304
  }}));
285
305
  process.exit(1);
@@ -292,10 +312,11 @@ class BunEngine(Engine):
292
312
  runtime_path = temp_dir / "runtime.js"
293
313
  with open(runtime_path, "w") as f:
294
314
  f.write(self.runtime_template)
315
+ logger.debug("Runtime file written to: %s", runtime_path)
295
316
 
296
317
  # Symlink node_modules from shared directory
297
318
  os.symlink(self.modules_dir / "node_modules", temp_dir / "node_modules")
298
-
319
+ logger.debug("node_modules symlinked from: %s", self.modules_dir / "node_modules")
299
320
 
300
321
  def validate_environment(self) -> bool:
301
322
  # Check multiple locations for bun
@@ -335,22 +356,23 @@ class BunEngine(Engine):
335
356
  # Read user script
336
357
  with open(program_file, 'r') as f:
337
358
  user_script = f.read()
359
+ logger.debug("Loaded user script: %d characters", len(user_script))
338
360
 
339
361
  # Create wrapper script
340
362
  wrapper = self.create_wrapper(input_data, user_script)
341
-
342
363
  script_path = temp_dir / "script.js"
343
- with open(script_path, "w") as f:
364
+ with open(script_path, 'w') as f:
344
365
  f.write(wrapper)
366
+ logger.debug("Generated wrapper script: %s", script_path)
345
367
 
346
368
  # Execute with Bun
347
369
  process = subprocess.Popen(
348
370
  [
349
371
  self.get_bun_path(),
350
372
  "run",
351
- "--no-fetch", # Disable network
352
- "--smol", # Reduced memory
353
- "--silent", # Reduce Bun's own error noise
373
+ "--no-fetch",
374
+ "--smol",
375
+ "--silent",
354
376
  str(script_path)
355
377
  ],
356
378
  stdout=subprocess.PIPE,
@@ -358,36 +380,64 @@ class BunEngine(Engine):
358
380
  text=True,
359
381
  cwd=temp_dir
360
382
  )
383
+ logger.info("Started Bun process PID %d", process.pid)
361
384
 
362
385
  try:
363
- output, error = process.communicate(timeout=5) # 5 second timeout
386
+ output, error = process.communicate(timeout=5)
387
+ logger.debug("Process completed with code %d", process.returncode)
364
388
  except subprocess.TimeoutExpired as e:
389
+ logger.warning("Process timeout after 5 seconds")
365
390
  process.kill()
366
- raise ProcessingError("Script execution timed out") from e
391
+ return {"lam.error": "Script execution timed out"}, "Execution timed out after 5 seconds"
367
392
 
368
- if error:
393
+ # Handle process errors
394
+ if process.returncode != 0:
369
395
  try:
370
- error_data = json.loads(error)
396
+ # Try to parse structured error from stderr
397
+ error_data = json.loads(error.strip())
371
398
  error_msg = error_data.get('error', 'Unknown error')
372
- if error_data.get('stack'):
373
- error_msg = f"{error_msg}\nStack trace:\n{error_data['stack']}"
399
+ stack = error_data.get('stack', [])
400
+
401
+ # Format error message
402
+ error_details = {
403
+ "lam.error": error_msg,
404
+ "stack_trace": stack
405
+ }
406
+ return error_details, error_msg
407
+
374
408
  except json.JSONDecodeError:
375
- error_msg = error.split('\n')[0] # Just take the first line if not JSON
376
- raise ProcessingError(error_msg)
409
+ # Fallback to raw error output
410
+ error_msg = error.strip() or "Unknown error"
411
+ return {"lam.error": error_msg}, error_msg
377
412
 
413
+ # Handle successful output
378
414
  try:
379
- return json.loads(output), None
380
- except json.JSONDecodeError:
381
- return {"lam.result": output}, None
415
+ output_data = json.loads(output)
416
+
417
+ # Process JavaScript logs (if any)
418
+ if 'logs' in output_data:
419
+ for log_entry in output_data.get('logs', []):
420
+ if log_entry['type'] == 'error':
421
+ logger.error("[JS] %s", log_entry['message'])
422
+ else:
423
+ logger.debug("[JS] %s", log_entry['message'])
424
+
425
+ result = output_data.get('result', {})
426
+ return result, None
427
+
428
+ except json.JSONDecodeError as e:
429
+ logger.error("Failed to parse output: %s", str(e))
430
+ return {
431
+ "lam.error": "Invalid JSON output",
432
+ "raw_output": output.strip()
433
+ }, "Output format error"
382
434
 
383
435
  except Exception as e:
384
- stats_data = stats.finalize()
385
- self.track_event('lam.bun.error', {
386
- 'error': str(e),
387
- 'error_type': e.__class__.__name__,
388
- **stats_data
389
- })
390
- return {"lam.error": str(e)}, str(e)
436
+ logger.exception("Execution failed")
437
+ return {
438
+ "lam.error": str(e),
439
+ "type": e.__class__.__name__
440
+ }, str(e)
391
441
 
392
442
  def get_engine(engine_type: str, workspace_id: str, flow_id: str, execution_id: str) -> Engine:
393
443
  """Factory function to get the appropriate execution engine"""
@@ -409,13 +459,16 @@ def get_engine(engine_type: str, workspace_id: str, flow_id: str, execution_id:
409
459
  def process_input(input: str) -> Tuple[str, Optional[str]]:
410
460
  """Process and validate input data"""
411
461
  if os.path.isfile(input):
462
+ logger.debug("Loading input from file: %s", input)
412
463
  with open(input, 'r') as file:
413
464
  return file.read(), None
414
465
 
415
466
  try:
416
467
  json.loads(input)
468
+ logger.debug("Validated inline JSON input")
417
469
  return input, None
418
470
  except json.JSONDecodeError as e:
471
+ logger.error("Invalid JSON input: %s", str(e))
419
472
  return None, str(e)
420
473
 
421
474
  @click.group()
@@ -435,16 +488,14 @@ def lam():
435
488
  def run(program_file: str, input: str, language: str, workspace_id: str,
436
489
  flow_id: str, execution_id: str, as_json: bool):
437
490
  """Execute a LAM transformation script"""
438
- stats = Stats() # Start tracking stats at the top level
491
+ stats = Stats()
439
492
 
440
- # Initialize engine
441
493
  try:
442
494
  engine = get_engine(language, workspace_id, flow_id, execution_id)
443
495
  except (ValueError, EnvironmentError) as e:
444
496
  click.echo({"lam.error": str(e)}, err=True)
445
497
  return
446
498
 
447
- # Setup logging
448
499
  log_file = engine.get_log_file()
449
500
  result_file = engine.get_result_file()
450
501
 
@@ -452,25 +503,22 @@ def run(program_file: str, input: str, language: str, workspace_id: str,
452
503
  file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
453
504
  logger.addHandler(file_handler)
454
505
 
455
- logger.info(f"Starting LAM execution with {language} engine")
506
+ logger.info("Starting LAM execution with %s engine", language)
456
507
  engine.track_event('lam.run.start', {
457
508
  'language': language,
458
509
  'program_file': program_file
459
510
  })
460
511
 
461
512
  try:
462
- # Process input
463
513
  input_data, error = process_input(input)
464
514
  if error:
465
515
  raise ProcessingError(f"Invalid input: {error}")
466
516
 
467
- # Execute transformation
468
517
  result, error = engine.execute(program_file, input_data)
469
518
 
470
- # Get final stats
471
519
  stats_data = stats.finalize()
472
- logger.info(f"Execution stats: duration={stats_data['duration_ms']:.2f}ms, "
473
- f"memory_used={stats_data['memory_used_mb']:.2f}MB")
520
+ logger.info("Execution stats: duration=%.2fms, memory=%.2fMB",
521
+ stats_data['duration_ms'], stats_data['memory_used_mb'])
474
522
 
475
523
  if error:
476
524
  click.echo({"lam.error": error}, err=True)
@@ -480,20 +528,17 @@ def run(program_file: str, input: str, language: str, workspace_id: str,
480
528
  click.echo(output)
481
529
  engine.track_event('lam.run.success', stats_data)
482
530
 
483
- # Save result with stats
484
- result_with_stats = {
485
- 'result': result,
486
- 'stats': stats_data,
487
- 'error': error or None
488
- }
531
+ if isinstance(result, list):
532
+ result = {"lam.result": result}
533
+
489
534
  with open(result_file, 'w') as f:
490
- json.dump(result_with_stats, f, indent=4)
535
+ json.dump(result, f, indent=4)
491
536
 
492
537
  except Exception as e:
493
538
  stats_data = stats.finalize()
494
- logger.error(f"Execution failed: {e}")
495
- logger.error(f"Final stats: duration={stats_data['duration_ms']:.2f}ms, "
496
- f"memory_used={stats_data['memory_used_mb']:.2f}MB")
539
+ logger.error("Execution failed: %s", str(e))
540
+ logger.error("Final stats: duration=%.2fms, memory=%.2fMB",
541
+ stats_data['duration_ms'], stats_data['memory_used_mb'])
497
542
  click.echo({"lam.error": str(e)}, err=True)
498
543
  engine.track_event('lam.run.error', {'error': str(e), **stats_data})
499
544
 
@@ -0,0 +1,52 @@
1
+ Metadata-Version: 2.2
2
+ Name: lam-cli
3
+ Version: 0.1.3
4
+ Summary: Secure data transformation tool supporting JQ and JavaScript (Bun)
5
+ Home-page: https://github.com/laminar-run/lam
6
+ Author: Laminar Run, Inc.
7
+ Author-email: connect@laminar.run
8
+ License: GPLv3
9
+ Project-URL: Documentation, https://docs.laminar.run
10
+ Project-URL: Source, https://github.com/laminar-run/lam
11
+ Project-URL: Issue Tracker, https://github.com/laminar-run/lam/issues
12
+ Keywords: laminar,api,integration,transformation,json,jq,javascript,bun
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Environment :: Console
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3.9
20
+ Classifier: Programming Language :: Python :: 3.10
21
+ Classifier: Programming Language :: Python :: 3.11
22
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
+ Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
24
+ Classifier: Topic :: Software Development :: Build Tools
25
+ Requires-Python: >=3.9
26
+ Description-Content-Type: text/markdown
27
+ License-File: LICENSE
28
+ Requires-Dist: backoff>=2.2.1
29
+ Requires-Dist: certifi>=2024.12.14
30
+ Requires-Dist: charset-normalizer>=3.3.2
31
+ Requires-Dist: click>=8.1.7
32
+ Requires-Dist: idna>=3.7
33
+ Requires-Dist: logtail-python>=0.2.2
34
+ Requires-Dist: monotonic>=1.6
35
+ Requires-Dist: msgpack>=1.0.8
36
+ Requires-Dist: posthog>=3.4.0
37
+ Requires-Dist: psutil>=5.9.0
38
+ Requires-Dist: python-dateutil>=2.8.2
39
+ Requires-Dist: requests>=2.32.3
40
+ Requires-Dist: six>=1.16.0
41
+ Requires-Dist: urllib3>=2.2.2
42
+ Dynamic: author
43
+ Dynamic: author-email
44
+ Dynamic: classifier
45
+ Dynamic: description-content-type
46
+ Dynamic: home-page
47
+ Dynamic: keywords
48
+ Dynamic: license
49
+ Dynamic: project-url
50
+ Dynamic: requires-dist
51
+ Dynamic: requires-python
52
+ Dynamic: summary
@@ -0,0 +1,8 @@
1
+ lam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ lam/lam.py,sha256=MrJKA_tFVEEvnhKJkMtrDbdL50bX27aJ0WUbP2dUJiU,20981
3
+ lam_cli-0.1.3.dist-info/LICENSE,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
4
+ lam_cli-0.1.3.dist-info/METADATA,sha256=urxYh-mNjUh3v-ggzB_gDIdp5JOa1aQPVKx9kPQGSio,1895
5
+ lam_cli-0.1.3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
6
+ lam_cli-0.1.3.dist-info/entry_points.txt,sha256=iJSsJitcGMikKJ3Q9KNWxEy911oTrSxGSO1HiwcBrKE,40
7
+ lam_cli-0.1.3.dist-info/top_level.txt,sha256=WyM7-Ig60qQH9meqS293pEd83jrMtbvGJM8ALZOQCtA,4
8
+ lam_cli-0.1.3.dist-info/RECORD,,
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ laminar = lam.lam:lam
@@ -1,196 +0,0 @@
1
- Metadata-Version: 2.2
2
- Name: lam-cli
3
- Version: 0.1.0
4
- Summary: Secure data transformation tool supporting JQ and JavaScript (Bun)
5
- Home-page: https://github.com/laminar-run/lam
6
- Author: Laminar Run, Inc.
7
- Author-email: connect@laminar.run
8
- License: GPLv3
9
- Project-URL: Documentation, https://docs.laminar.run
10
- Project-URL: Source, https://github.com/laminar-run/lam
11
- Project-URL: Issue Tracker, https://github.com/laminar-run/lam/issues
12
- Keywords: laminar,api,integration,transformation,json,jq,javascript,bun
13
- Classifier: Development Status :: 4 - Beta
14
- Classifier: Environment :: Console
15
- Classifier: Intended Audience :: Developers
16
- Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
17
- Classifier: Operating System :: OS Independent
18
- Classifier: Programming Language :: Python :: 3
19
- Classifier: Programming Language :: Python :: 3.9
20
- Classifier: Programming Language :: Python :: 3.10
21
- Classifier: Programming Language :: Python :: 3.11
22
- Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
- Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
24
- Classifier: Topic :: Software Development :: Build Tools
25
- Requires-Python: >=3.9
26
- Description-Content-Type: text/markdown
27
- License-File: LICENSE
28
- Requires-Dist: backoff>=2.2.1
29
- Requires-Dist: certifi>=2024.12.14
30
- Requires-Dist: charset-normalizer>=3.3.2
31
- Requires-Dist: click>=8.1.7
32
- Requires-Dist: idna>=3.7
33
- Requires-Dist: logtail-python>=0.2.2
34
- Requires-Dist: monotonic>=1.6
35
- Requires-Dist: msgpack>=1.0.8
36
- Requires-Dist: posthog>=3.4.0
37
- Requires-Dist: psutil>=5.9.0
38
- Requires-Dist: python-dateutil>=2.8.2
39
- Requires-Dist: requests>=2.32.3
40
- Requires-Dist: six>=1.16.0
41
- Requires-Dist: urllib3>=2.2.2
42
- Dynamic: author
43
- Dynamic: author-email
44
- Dynamic: classifier
45
- Dynamic: description
46
- Dynamic: description-content-type
47
- Dynamic: home-page
48
- Dynamic: keywords
49
- Dynamic: license
50
- Dynamic: project-url
51
- Dynamic: requires-dist
52
- Dynamic: requires-python
53
- Dynamic: summary
54
-
55
- # lam
56
- Lam is a data transformation tool for Laminar that supports both `jq` and JavaScript transformations using Bun.
57
-
58
- ## Quickstart
59
- Install the dependencies:
60
- ```bash
61
- # For JQ support
62
- brew install jq # or sudo apt-get install jq
63
-
64
- # For JavaScript support
65
- curl -fsSL https://bun.sh/install | bash
66
-
67
- make setup
68
- ```
69
-
70
- Run the CLI tool:
71
- ```bash
72
- make cli ARGS="run <program> <input> [--language jq|js]"
73
- ```
74
-
75
- ## Features
76
- - JQ transformations (default)
77
- - JavaScript transformations with Bun runtime
78
- - Built-in utilities (lodash, date-fns)
79
- - Resource monitoring and limits
80
- - Detailed execution statistics
81
- - Secure execution environment
82
-
83
- ## Examples
84
-
85
- ### JQ Transform
86
- ```bash
87
- make cli ARGS="run examples/transform.jq data.json"
88
- ```
89
-
90
- ### JavaScript Transform
91
- ```bash
92
- make cli ARGS="run examples/transform.js data.json --language js"
93
- ```
94
-
95
- Example JavaScript transform:
96
- ```javascript
97
- (input) => {
98
- // Lodash available as _
99
- return _.map(input.data, item => ({
100
- value: item.value * 2
101
- }));
102
- }
103
- ```
104
-
105
- ## Installation
106
-
107
- ### Docker Installation
108
- ```dockerfile
109
- # Install lam-cli
110
- RUN pip3 install git+https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/user/project.git@{version}
111
-
112
- # Install dependencies
113
- RUN apt-get update && apt-get install -y jq
114
- RUN curl -fsSL https://bun.sh/install | bash
115
- ```
116
-
117
- ### Manual Setup
118
- Create a virtual environment and install dependencies:
119
- ```bash
120
- python3 -m venv ./venv
121
- source ./venv/bin/activate
122
- pip install -r requirements.txt
123
- ```
124
-
125
- ## Usage
126
- ```bash
127
- # Basic usage
128
- python3 ./lam/lam.py run <program> <input>
129
-
130
- # With JavaScript
131
- python3 ./lam/lam.py run script.js data.json --language js
132
-
133
- # Full options
134
- python3 ./lam/lam.py run <program> <input> \
135
- --language [jq|js] \
136
- --workspace_id <id> \
137
- --flow_id <id> \
138
- --execution_id <id> \
139
- [--as-json]
140
- ```
141
-
142
- ## Resource Limits
143
- - Maximum input size: 10MB
144
- - Execution timeout: 5 seconds
145
- - Memory limits enabled
146
- - Disk space monitoring
147
-
148
- ## Security
149
- - Sandboxed JavaScript execution
150
- - Network access disabled
151
- - Limited global scope
152
- - Resource monitoring
153
- - Secure dependency management
154
-
155
- ## Logging and Monitoring
156
- - Execution statistics (duration, memory usage)
157
- - Detailed error tracking
158
- - PostHog analytics integration
159
- - Log file generation
160
-
161
- ## Development
162
- ```bash
163
- # Run all tests
164
- make test
165
-
166
- # Run specific test suite
167
- make test-jq
168
- make test-js
169
- make test-js-edge-cases
170
-
171
- # Run single test
172
- make test-single TEST=test/js/example.js DATA=test/data/input.json
173
- ```
174
-
175
- ## Releases
176
- Update version in `setup.py`:
177
- ```python
178
- setup(
179
- name="lam-cli",
180
- version="0.0.<x>",
181
- ...
182
- )
183
- ```
184
-
185
- Create and push tag:
186
- ```bash
187
- git tag v<version>-<increment>
188
- git push origin v<version>-<increment>
189
- ```
190
-
191
- ## Dependencies
192
- Update dependencies:
193
- ```bash
194
- pip3 install <package>
195
- pip3 freeze > requirements.txt
196
- ```
@@ -1,8 +0,0 @@
1
- lam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- lam/lam.py,sha256=22zXRw3pz0yQvsORvpWIdUjmvaFhP_6_DMkw02_CG1o,17939
3
- lam_cli-0.1.0.dist-info/LICENSE,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
4
- lam_cli-0.1.0.dist-info/METADATA,sha256=z2GeyYTNb3x8gM4Px8XUlldd8UpC86ZyUunGeO6SeQM,4674
5
- lam_cli-0.1.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
6
- lam_cli-0.1.0.dist-info/entry_points.txt,sha256=ph7QV6H2VWqf9fU5rtoAgEabDgZ4f85ZImdLXeBmdfA,36
7
- lam_cli-0.1.0.dist-info/top_level.txt,sha256=WyM7-Ig60qQH9meqS293pEd83jrMtbvGJM8ALZOQCtA,4
8
- lam_cli-0.1.0.dist-info/RECORD,,
@@ -1,2 +0,0 @@
1
- [console_scripts]
2
- lam = lam.lam:lam