langsmith-cli 0.3.2__tar.gz → 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/.claude-plugin/plugin.json +1 -1
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/CLAUDE.md +36 -11
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/PKG-INFO +19 -7
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/README.md +18 -6
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/QOL_IMPROVEMENTS.md +0 -1
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/MCP_PARITY.md +0 -1
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/pyproject.toml +1 -1
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/SKILL.md +11 -3
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/docs/examples.md +3 -7
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/references/runs.md +0 -2
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/references/troubleshooting.md +8 -2
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/commands/datasets.py +14 -3
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/commands/prompts.py +6 -1
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/commands/runs.py +29 -20
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/main.py +35 -2
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/utils.py +183 -11
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_datasets.py +31 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_main.py +213 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_prompts.py +18 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_runs_analyze.py +17 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_runs_discovery.py +12 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_runs_get.py +22 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_runs_list.py +91 -24
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_runs_roots.py +11 -18
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_runs_sample.py +27 -1
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_runs_search.py +11 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_utils.py +364 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/.claude-plugin/marketplace.json +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/.env.example +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/.github/dependabot.yml +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/.github/workflows/ci.yml +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/.github/workflows/dependency-review.yml +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/.github/workflows/publish.yml +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/.gitignore +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/.pre-commit-config.yaml +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/.python-version +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/AGENTS.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/LICENSE +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/RELEASING.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/COMMANDS_DESIGN.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/PIPES_TO_CLI_REFERENCE.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/PRD.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/QOL_FEATURES.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/TLDR.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/CI_BEST_PRACTICES.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/CODECOV_SETUP.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/IMPLEMENTATION_PLAN.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/LANGSMITH_TEAM_QUESTIONS.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/PUBLISHING.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/PYPI_SETUP_SUMMARY.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/SESSION_DIRECTIVES.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/TESTING_PERFORMANCE.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/TESTING_STRATEGY.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/docs/dev/TYPE_SAFETY_GUIDE.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/main.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/scripts/install.ps1 +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/scripts/install.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/scripts/install.sh +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/scripts/release.sh +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/scripts/test_installer.sh +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/scripts/uninstall.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/docs/reference.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/references/datasets.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/references/examples.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/references/fql.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/references/installation.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/references/projects.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/skills/langsmith/references/prompts.md +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/__init__.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/cli_logging.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/commands/auth.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/commands/examples.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/commands/projects.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/commands/self_cmd.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/config.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/field_analysis.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/src/langsmith_cli/filters.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/conftest.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_auth.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_config.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_e2e.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_examples.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_fetch_helpers.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_field_analysis.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_filters.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_logging.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_output_flag.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_projects.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_runs_fields.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_runs_view.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_self.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/tests/test_smoke.py +0 -0
- {langsmith_cli-0.3.2 → langsmith_cli-0.3.4}/uv.lock +0 -0
|
@@ -111,10 +111,18 @@ main.py (entry point)
|
|
|
111
111
|
├── runs (group)
|
|
112
112
|
│ ├── list
|
|
113
113
|
│ ├── get
|
|
114
|
+
│ ├── get-latest
|
|
114
115
|
│ ├── stats
|
|
115
116
|
│ ├── open
|
|
116
117
|
│ ├── watch
|
|
117
|
-
│
|
|
118
|
+
│ ├── search
|
|
119
|
+
│ ├── sample
|
|
120
|
+
│ ├── analyze
|
|
121
|
+
│ ├── tags
|
|
122
|
+
│ ├── metadata-keys
|
|
123
|
+
│ ├── fields
|
|
124
|
+
│ ├── describe
|
|
125
|
+
│ └── view-file
|
|
118
126
|
├── datasets (group)
|
|
119
127
|
│ ├── list
|
|
120
128
|
│ ├── get
|
|
@@ -124,10 +132,13 @@ main.py (entry point)
|
|
|
124
132
|
│ ├── list
|
|
125
133
|
│ ├── get
|
|
126
134
|
│ └── create
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
135
|
+
├── prompts (group)
|
|
136
|
+
│ ├── list
|
|
137
|
+
│ ├── get
|
|
138
|
+
│ └── push
|
|
139
|
+
└── self (group)
|
|
140
|
+
├── detect
|
|
141
|
+
└── update
|
|
131
142
|
```
|
|
132
143
|
|
|
133
144
|
### Key Design Patterns
|
|
@@ -217,23 +228,37 @@ def list_runs(ctx, output_format, count, output, ...):
|
|
|
217
228
|
```
|
|
218
229
|
src/langsmith_cli/
|
|
219
230
|
├── __init__.py
|
|
220
|
-
├── main.py # Entry point, CLI group registration
|
|
221
|
-
├──
|
|
231
|
+
├── main.py # Entry point, CLI group registration, global error handler
|
|
232
|
+
├── cli_logging.py # CLILogger for verbosity control and stream separation
|
|
233
|
+
├── config.py # Credentials file management
|
|
234
|
+
├── field_analysis.py # Field discovery and statistics (runs fields/describe)
|
|
235
|
+
├── filters.py # FQL filter builders and time parsing
|
|
236
|
+
├── utils.py # Shared helpers (output formatting, project resolution, etc.)
|
|
222
237
|
└── commands/ # Modular command implementations
|
|
223
238
|
├── auth.py # Authentication (login)
|
|
224
239
|
├── projects.py # Project management
|
|
225
240
|
├── runs.py # Runs/traces (largest module)
|
|
226
241
|
├── datasets.py # Dataset operations
|
|
227
242
|
├── examples.py # Dataset examples
|
|
228
|
-
|
|
243
|
+
├── prompts.py # Prompt management
|
|
244
|
+
└── self_cmd.py # Self-inspection (detect, update)
|
|
229
245
|
|
|
230
246
|
tests/
|
|
231
|
-
├── conftest.py # Pytest fixtures (CliRunner)
|
|
232
|
-
├── test_main.py # Root CLI tests
|
|
247
|
+
├── conftest.py # Pytest fixtures (CliRunner, model factories)
|
|
248
|
+
├── test_main.py # Root CLI + global error handler tests
|
|
233
249
|
├── test_logging.py # CLILogger tests
|
|
234
250
|
├── test_auth.py # Auth command tests
|
|
235
251
|
├── test_projects.py # Projects command tests
|
|
236
|
-
├── test_runs.py # Runs command tests
|
|
252
|
+
├── test_runs.py # Runs command tests
|
|
253
|
+
├── test_runs_list.py # Runs list command tests (filters, JSON, formats)
|
|
254
|
+
├── test_runs_get.py # Runs get/get-latest/open tests
|
|
255
|
+
├── test_runs_roots.py # Runs --roots flag tests
|
|
256
|
+
├── test_runs_sample.py # Runs sample command tests
|
|
257
|
+
├── test_datasets.py # Datasets command tests
|
|
258
|
+
├── test_examples.py # Examples command tests
|
|
259
|
+
├── test_prompts.py # Prompts command tests
|
|
260
|
+
├── test_self.py # Self detect/update command tests
|
|
261
|
+
├── test_utils.py # Utility function tests
|
|
237
262
|
├── test_smoke.py # Smoke tests (requires API key)
|
|
238
263
|
└── test_e2e.py # End-to-end tests (requires API key)
|
|
239
264
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: langsmith-cli
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.4
|
|
4
4
|
Summary: Context-efficient CLI for LangSmith. Built for humans and agents.
|
|
5
5
|
Project-URL: Homepage, https://github.com/aviadr1/langsmith-cli
|
|
6
6
|
Project-URL: Repository, https://github.com/aviadr1/langsmith-cli
|
|
@@ -158,10 +158,11 @@ langsmith-cli runs watch
|
|
|
158
158
|
### 📦 **Complete Coverage**
|
|
159
159
|
Every LangSmith resource at your fingertips:
|
|
160
160
|
- ✅ **Projects** - List, create, inspect
|
|
161
|
-
- ✅ **Runs** - Search, stats, watch,
|
|
161
|
+
- ✅ **Runs** - Search, stats, watch, sample, analyze, field discovery
|
|
162
162
|
- ✅ **Datasets** - CRUD + bulk JSONL uploads
|
|
163
163
|
- ✅ **Examples** - Full lifecycle management
|
|
164
164
|
- ✅ **Prompts** - Version control your prompts
|
|
165
|
+
- ✅ **Self** - Installation detection + auto-update
|
|
165
166
|
|
|
166
167
|
---
|
|
167
168
|
|
|
@@ -310,8 +311,8 @@ langsmith-cli runs watch --project production
|
|
|
310
311
|
### 💾 Bulk Dataset Uploads
|
|
311
312
|
|
|
312
313
|
```bash
|
|
313
|
-
# Export examples to JSONL
|
|
314
|
-
langsmith-cli
|
|
314
|
+
# Export examples to JSONL (using --output for reliable file writing)
|
|
315
|
+
langsmith-cli examples list --dataset my-dataset --output examples.jsonl
|
|
315
316
|
|
|
316
317
|
# Upload to new dataset
|
|
317
318
|
langsmith-cli datasets push examples.jsonl --dataset production-eval
|
|
@@ -424,9 +425,18 @@ auth login # Authenticate with LangSmith
|
|
|
424
425
|
projects list # List all projects
|
|
425
426
|
runs list # Search and filter runs
|
|
426
427
|
runs get <id> # Inspect a specific run
|
|
428
|
+
runs get-latest # Get most recent run matching filters
|
|
427
429
|
runs stats # Aggregate statistics
|
|
428
430
|
runs watch # Live run dashboard
|
|
429
431
|
runs open <id> # Open trace in browser
|
|
432
|
+
runs search # Full-text search across runs
|
|
433
|
+
runs sample # Stratified sampling by tags/metadata
|
|
434
|
+
runs analyze # Group runs and compute metrics
|
|
435
|
+
runs tags # Discover tag patterns
|
|
436
|
+
runs metadata-keys # Discover metadata keys
|
|
437
|
+
runs fields # Discover field paths and types
|
|
438
|
+
runs describe # Detailed field statistics
|
|
439
|
+
runs view-file # View runs from JSONL files
|
|
430
440
|
datasets list # List datasets
|
|
431
441
|
datasets create # Create new dataset
|
|
432
442
|
datasets push # Bulk upload from JSONL
|
|
@@ -435,6 +445,8 @@ examples create # Add example to dataset
|
|
|
435
445
|
prompts list # List prompt repositories
|
|
436
446
|
prompts get # Pull a prompt template
|
|
437
447
|
prompts push # Push local prompt to LangSmith
|
|
448
|
+
self detect # Show installation details
|
|
449
|
+
self update # Update to latest version
|
|
438
450
|
```
|
|
439
451
|
|
|
440
452
|
### Global Flags
|
|
@@ -450,7 +462,8 @@ prompts push # Push local prompt to LangSmith
|
|
|
450
462
|
|
|
451
463
|
| Option | Description | Example |
|
|
452
464
|
|--------|-------------|---------|
|
|
453
|
-
| `--project` | Filter by project | `--project production` |
|
|
465
|
+
| `--project` | Filter by project name | `--project production` |
|
|
466
|
+
| `--project-id` | Filter by project UUID | `--project-id abc-123...` |
|
|
454
467
|
| `--status` | Filter by status | `--status error` |
|
|
455
468
|
| `--failed` | Only failed runs | `--failed` |
|
|
456
469
|
| `--succeeded` | Only successful runs | `--succeeded` |
|
|
@@ -542,8 +555,7 @@ uv run pyright
|
|
|
542
555
|
```
|
|
543
556
|
|
|
544
557
|
### Project Stats
|
|
545
|
-
- **
|
|
546
|
-
- **100% Utils Coverage** (47 tests for helpers)
|
|
558
|
+
- **92% Test Coverage** (589 tests)
|
|
547
559
|
- **Zero Type Errors** (Pyright clean)
|
|
548
560
|
- **100% MCP Parity** (13/13 tools)
|
|
549
561
|
|
|
@@ -108,10 +108,11 @@ langsmith-cli runs watch
|
|
|
108
108
|
### 📦 **Complete Coverage**
|
|
109
109
|
Every LangSmith resource at your fingertips:
|
|
110
110
|
- ✅ **Projects** - List, create, inspect
|
|
111
|
-
- ✅ **Runs** - Search, stats, watch,
|
|
111
|
+
- ✅ **Runs** - Search, stats, watch, sample, analyze, field discovery
|
|
112
112
|
- ✅ **Datasets** - CRUD + bulk JSONL uploads
|
|
113
113
|
- ✅ **Examples** - Full lifecycle management
|
|
114
114
|
- ✅ **Prompts** - Version control your prompts
|
|
115
|
+
- ✅ **Self** - Installation detection + auto-update
|
|
115
116
|
|
|
116
117
|
---
|
|
117
118
|
|
|
@@ -260,8 +261,8 @@ langsmith-cli runs watch --project production
|
|
|
260
261
|
### 💾 Bulk Dataset Uploads
|
|
261
262
|
|
|
262
263
|
```bash
|
|
263
|
-
# Export examples to JSONL
|
|
264
|
-
langsmith-cli
|
|
264
|
+
# Export examples to JSONL (using --output for reliable file writing)
|
|
265
|
+
langsmith-cli examples list --dataset my-dataset --output examples.jsonl
|
|
265
266
|
|
|
266
267
|
# Upload to new dataset
|
|
267
268
|
langsmith-cli datasets push examples.jsonl --dataset production-eval
|
|
@@ -374,9 +375,18 @@ auth login # Authenticate with LangSmith
|
|
|
374
375
|
projects list # List all projects
|
|
375
376
|
runs list # Search and filter runs
|
|
376
377
|
runs get <id> # Inspect a specific run
|
|
378
|
+
runs get-latest # Get most recent run matching filters
|
|
377
379
|
runs stats # Aggregate statistics
|
|
378
380
|
runs watch # Live run dashboard
|
|
379
381
|
runs open <id> # Open trace in browser
|
|
382
|
+
runs search # Full-text search across runs
|
|
383
|
+
runs sample # Stratified sampling by tags/metadata
|
|
384
|
+
runs analyze # Group runs and compute metrics
|
|
385
|
+
runs tags # Discover tag patterns
|
|
386
|
+
runs metadata-keys # Discover metadata keys
|
|
387
|
+
runs fields # Discover field paths and types
|
|
388
|
+
runs describe # Detailed field statistics
|
|
389
|
+
runs view-file # View runs from JSONL files
|
|
380
390
|
datasets list # List datasets
|
|
381
391
|
datasets create # Create new dataset
|
|
382
392
|
datasets push # Bulk upload from JSONL
|
|
@@ -385,6 +395,8 @@ examples create # Add example to dataset
|
|
|
385
395
|
prompts list # List prompt repositories
|
|
386
396
|
prompts get # Pull a prompt template
|
|
387
397
|
prompts push # Push local prompt to LangSmith
|
|
398
|
+
self detect # Show installation details
|
|
399
|
+
self update # Update to latest version
|
|
388
400
|
```
|
|
389
401
|
|
|
390
402
|
### Global Flags
|
|
@@ -400,7 +412,8 @@ prompts push # Push local prompt to LangSmith
|
|
|
400
412
|
|
|
401
413
|
| Option | Description | Example |
|
|
402
414
|
|--------|-------------|---------|
|
|
403
|
-
| `--project` | Filter by project | `--project production` |
|
|
415
|
+
| `--project` | Filter by project name | `--project production` |
|
|
416
|
+
| `--project-id` | Filter by project UUID | `--project-id abc-123...` |
|
|
404
417
|
| `--status` | Filter by status | `--status error` |
|
|
405
418
|
| `--failed` | Only failed runs | `--failed` |
|
|
406
419
|
| `--succeeded` | Only successful runs | `--succeeded` |
|
|
@@ -492,8 +505,7 @@ uv run pyright
|
|
|
492
505
|
```
|
|
493
506
|
|
|
494
507
|
### Project Stats
|
|
495
|
-
- **
|
|
496
|
-
- **100% Utils Coverage** (47 tests for helpers)
|
|
508
|
+
- **92% Test Coverage** (589 tests)
|
|
497
509
|
- **Zero Type Errors** (Pyright clean)
|
|
498
510
|
- **100% MCP Parity** (13/13 tools)
|
|
499
511
|
|
|
@@ -45,7 +45,6 @@ This document tracks feature parity between the langsmith-cli and the official L
|
|
|
45
45
|
- ✅ `is_root` → `--is-root`
|
|
46
46
|
- ✅ `trace_filter` → `--trace-filter`
|
|
47
47
|
- ✅ `tree_filter` → `--tree-filter`
|
|
48
|
-
- ✅ `order_by` → `--order-by`
|
|
49
48
|
- ✅ `reference_example_id` → `--reference-example-id`
|
|
50
49
|
|
|
51
50
|
**`runs get`** (2/2 parameters):
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "langsmith-cli"
|
|
3
3
|
# IMPORTANT: When bumping this version, also update .claude-plugin/plugin.json
|
|
4
|
-
version = "0.3.
|
|
4
|
+
version = "0.3.4"
|
|
5
5
|
description = "Context-efficient CLI for LangSmith. Built for humans and agents."
|
|
6
6
|
readme = "README.md"
|
|
7
7
|
requires-python = ">=3.12"
|
|
@@ -48,8 +48,8 @@ langsmith-cli runs list --project my-project --fields id,name,status --output ru
|
|
|
48
48
|
```bash
|
|
49
49
|
# ❌ WRONG - Never use shell redirection for data extraction
|
|
50
50
|
langsmith-cli --json runs list --project my-project > runs.json
|
|
51
|
-
# If API fails:
|
|
52
|
-
# You won't know
|
|
51
|
+
# If API fails: errors go to stderr (invisible with redirection)
|
|
52
|
+
# You may get a JSON error object instead of data, and won't know what happened
|
|
53
53
|
```
|
|
54
54
|
|
|
55
55
|
```bash
|
|
@@ -107,7 +107,8 @@ langsmith-cli --json runs list --project my-project --limit 5 2>&1
|
|
|
107
107
|
|
|
108
108
|
### Runs (Traces)
|
|
109
109
|
- `langsmith-cli --json runs list [OPTIONS]`: List recent runs.
|
|
110
|
-
- `--project <name>`: Filter by project.
|
|
110
|
+
- `--project <name>`: Filter by project name (default: "default").
|
|
111
|
+
- `--project-id <uuid>`: Filter by project UUID (bypasses name resolution, faster).
|
|
111
112
|
- `--limit <n>`: Max results (default 10, keep it small).
|
|
112
113
|
- `--status <success|error>`: Filter by status.
|
|
113
114
|
- `--filter <string>`: Advanced FQL query string (see FQL examples below).
|
|
@@ -211,6 +212,13 @@ langsmith-cli --json runs list --project my-project --limit 5 2>&1
|
|
|
211
212
|
- `langsmith-cli --json prompts get <name> [--commit <hash>]`: Fetch a prompt template.
|
|
212
213
|
- `langsmith-cli --json prompts push <name> <file_path>`: Push a local file as a prompt.
|
|
213
214
|
|
|
215
|
+
### Self (Installation Management)
|
|
216
|
+
- `langsmith-cli self detect`: Show installation details (version, install method, paths).
|
|
217
|
+
- Reports: version, install method (uv tool, pipx, pip, editable), install path, executable path, Python version.
|
|
218
|
+
- `langsmith-cli self update`: Update langsmith-cli to the latest version.
|
|
219
|
+
- Auto-detects install method and runs the appropriate upgrade command.
|
|
220
|
+
- Checks PyPI for latest version before updating.
|
|
221
|
+
|
|
214
222
|
## Common Patterns (No Piping Needed)
|
|
215
223
|
|
|
216
224
|
The CLI provides built-in commands that eliminate the need for Unix pipes, jq, and nested commands:
|
|
@@ -23,8 +23,7 @@ This document provides practical workflows and use cases for common LangSmith op
|
|
|
23
23
|
langsmith-cli --json runs list \
|
|
24
24
|
--project production-app \
|
|
25
25
|
--status error \
|
|
26
|
-
--limit 5
|
|
27
|
-
--order-by -start_time
|
|
26
|
+
--limit 5
|
|
28
27
|
|
|
29
28
|
# Step 2: Inspect specific failure (context-efficient)
|
|
30
29
|
langsmith-cli --json runs get <run-id> \
|
|
@@ -75,8 +74,7 @@ langsmith-cli runs open <trace-id>
|
|
|
75
74
|
langsmith-cli --json runs list \
|
|
76
75
|
--project production-app \
|
|
77
76
|
--filter 'gt(latency, "5s")' \
|
|
78
|
-
--limit 20
|
|
79
|
-
--order-by -latency
|
|
77
|
+
--limit 20
|
|
80
78
|
|
|
81
79
|
# Step 2: Analyze latency distribution
|
|
82
80
|
langsmith-cli --json runs stats \
|
|
@@ -374,8 +372,7 @@ langsmith-cli --json runs stats --project production-app --limit 1000
|
|
|
374
372
|
langsmith-cli --json runs list \
|
|
375
373
|
--project production-app \
|
|
376
374
|
--filter 'gt(total_cost, 0.1)' \
|
|
377
|
-
--limit 20
|
|
378
|
-
--order-by -total_cost
|
|
375
|
+
--limit 20
|
|
379
376
|
|
|
380
377
|
# Analyze cost by component type
|
|
381
378
|
langsmith-cli --json runs list \
|
|
@@ -685,7 +682,6 @@ while true; do
|
|
|
685
682
|
RUNS=$(langsmith-cli --json runs list \
|
|
686
683
|
--project "$PROJECT" \
|
|
687
684
|
--limit 100 \
|
|
688
|
-
--order-by -start_time \
|
|
689
685
|
$FILTER)
|
|
690
686
|
|
|
691
687
|
# Process each new run
|
|
@@ -18,7 +18,6 @@ langsmith-cli --json runs list [OPTIONS]
|
|
|
18
18
|
- `--filter TEXT` - Advanced FQL query (see Filter Query Language section)
|
|
19
19
|
- `--trace-filter TEXT` - Filter applied to root run of trace
|
|
20
20
|
- `--tree-filter TEXT` - Filter applied to any run in trace tree
|
|
21
|
-
- `--order-by TEXT` - Sort field (default: `-start_time`). Prefix with `-` for descending
|
|
22
21
|
- `--reference-example-id UUID` - Filter runs by reference example ID
|
|
23
22
|
|
|
24
23
|
**Output Fields:**
|
|
@@ -194,4 +193,3 @@ langsmith-cli runs watch [OPTIONS]
|
|
|
194
193
|
- `--refresh INTEGER` - Refresh interval in seconds (default: 2)
|
|
195
194
|
|
|
196
195
|
**Behavior:** Shows live table of recent runs with auto-refresh
|
|
197
|
-
|
|
@@ -80,8 +80,14 @@ langsmith-cli <command> [options]
|
|
|
80
80
|
**Solution:** Run `langsmith-cli auth login` or set `LANGSMITH_API_KEY` env var
|
|
81
81
|
|
|
82
82
|
### "Project not found"
|
|
83
|
-
**Cause:** Project name doesn't exist
|
|
84
|
-
**Solution:**
|
|
83
|
+
**Cause:** Project name doesn't exist or has a path prefix (e.g., `prd/my-project`)
|
|
84
|
+
**Solution:**
|
|
85
|
+
1. The error message will suggest similar project names if available
|
|
86
|
+
2. List projects: `langsmith-cli --json projects list --fields name`
|
|
87
|
+
3. Use substring matching: `langsmith-cli --json runs list --project-name my-project`
|
|
88
|
+
4. Use wildcards: `langsmith-cli --json runs list --project-name-pattern "*my-project*"`
|
|
89
|
+
5. If you have a UUID, pass it to `--project-id` or `--project` (UUIDs are auto-detected)
|
|
90
|
+
6. In JSON mode, the error includes structured `suggestions` and `failed_sources` fields
|
|
85
91
|
|
|
86
92
|
### "Dataset not found"
|
|
87
93
|
**Cause:** Dataset name doesn't match exactly
|
|
@@ -232,6 +232,17 @@ def push_dataset(ctx, file_path, dataset):
|
|
|
232
232
|
dataset_name=dataset,
|
|
233
233
|
)
|
|
234
234
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
235
|
+
if ctx.obj.get("json"):
|
|
236
|
+
click.echo(
|
|
237
|
+
json_dumps(
|
|
238
|
+
{
|
|
239
|
+
"status": "success",
|
|
240
|
+
"dataset": dataset,
|
|
241
|
+
"examples_count": len(examples),
|
|
242
|
+
}
|
|
243
|
+
)
|
|
244
|
+
)
|
|
245
|
+
else:
|
|
246
|
+
logger.success(
|
|
247
|
+
f"Successfully pushed {len(examples)} examples to dataset '{dataset}'"
|
|
248
|
+
)
|
|
@@ -164,4 +164,9 @@ def push_prompt(ctx, name, file_path, description, tags, is_public):
|
|
|
164
164
|
is_public=is_public,
|
|
165
165
|
)
|
|
166
166
|
|
|
167
|
-
|
|
167
|
+
if ctx.obj.get("json"):
|
|
168
|
+
from langsmith_cli.utils import json_dumps
|
|
169
|
+
|
|
170
|
+
click.echo(json_dumps({"status": "success", "name": name}))
|
|
171
|
+
else:
|
|
172
|
+
logger.success(f"Successfully pushed prompt to {name}")
|
|
@@ -25,11 +25,13 @@ from langsmith_cli.utils import (
|
|
|
25
25
|
filter_fields,
|
|
26
26
|
get_matching_items,
|
|
27
27
|
get_or_create_client,
|
|
28
|
+
get_project_suggestions,
|
|
28
29
|
json_dumps,
|
|
29
30
|
output_formatted_data,
|
|
30
31
|
output_option,
|
|
31
32
|
output_single_item,
|
|
32
33
|
parse_duration_to_seconds,
|
|
34
|
+
raise_if_all_failed_with_suggestions,
|
|
33
35
|
render_run_details,
|
|
34
36
|
resolve_project_filters,
|
|
35
37
|
sort_items,
|
|
@@ -321,9 +323,6 @@ def compute_metrics(
|
|
|
321
323
|
)
|
|
322
324
|
@click.option("--trace-filter", help="Filter applied to root trace.")
|
|
323
325
|
@click.option("--tree-filter", help="Filter if any run in trace tree matches.")
|
|
324
|
-
@click.option(
|
|
325
|
-
"--order-by", default="-start_time", help="Sort field (prefix with - for desc)."
|
|
326
|
-
)
|
|
327
326
|
@click.option("--reference-example-id", help="Filter runs for a specific example.")
|
|
328
327
|
@click.option(
|
|
329
328
|
"--tag",
|
|
@@ -430,7 +429,6 @@ def list_runs(
|
|
|
430
429
|
roots,
|
|
431
430
|
trace_filter,
|
|
432
431
|
tree_filter,
|
|
433
|
-
order_by,
|
|
434
432
|
reference_example_id,
|
|
435
433
|
tag,
|
|
436
434
|
name_pattern,
|
|
@@ -661,7 +659,6 @@ def list_runs(
|
|
|
661
659
|
is_root=is_root,
|
|
662
660
|
trace_filter=trace_filter,
|
|
663
661
|
tree_filter=tree_filter,
|
|
664
|
-
order_by=order_by,
|
|
665
662
|
reference_example_id=reference_example_id,
|
|
666
663
|
console=None, # Don't auto-report warnings (we have custom diagnostics below)
|
|
667
664
|
)
|
|
@@ -669,11 +666,10 @@ def list_runs(
|
|
|
669
666
|
failed_projects = result.failed_sources
|
|
670
667
|
|
|
671
668
|
# CRITICAL: Fail fast if ALL sources failed (prevents silent failures)
|
|
672
|
-
#
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
result.raise_if_all_failed(logger, "runs")
|
|
669
|
+
# The global error handler in LangSmithCLIGroup outputs JSON errors in --json mode,
|
|
670
|
+
# so we don't need to output [] here (which would cause double output on stdout).
|
|
671
|
+
# Uses raise_if_all_failed_with_suggestions to suggest similar project names.
|
|
672
|
+
raise_if_all_failed_with_suggestions(result, client, pq, logger, "runs")
|
|
677
673
|
|
|
678
674
|
# Report partial failures (some succeeded, some failed)
|
|
679
675
|
if result.has_failures:
|
|
@@ -993,7 +989,6 @@ def get_latest_run(
|
|
|
993
989
|
error=error_filter,
|
|
994
990
|
filter=combined_filter,
|
|
995
991
|
is_root=roots,
|
|
996
|
-
order_by="-start_time",
|
|
997
992
|
)
|
|
998
993
|
|
|
999
994
|
if pq.use_id:
|
|
@@ -1028,6 +1023,16 @@ def get_latest_run(
|
|
|
1028
1023
|
if len(failed_projects) > 3:
|
|
1029
1024
|
logger.warning(f" • ... and {len(failed_projects) - 3} more")
|
|
1030
1025
|
|
|
1026
|
+
# Suggest similar project names for single-project failures
|
|
1027
|
+
failed_names = [
|
|
1028
|
+
name for name, _ in failed_projects if not name.startswith("id:")
|
|
1029
|
+
]
|
|
1030
|
+
if len(failed_names) == 1:
|
|
1031
|
+
suggestions = get_project_suggestions(client, failed_names[0])
|
|
1032
|
+
if suggestions:
|
|
1033
|
+
suggestion_list = ", ".join(f"'{s}'" for s in suggestions[:5])
|
|
1034
|
+
logger.info(f"Did you mean: {suggestion_list}?")
|
|
1035
|
+
|
|
1031
1036
|
raise click.Abort()
|
|
1032
1037
|
|
|
1033
1038
|
data = filter_fields(latest_run, fields)
|
|
@@ -1168,7 +1173,14 @@ def run_stats(
|
|
|
1168
1173
|
resolved_project_ids.append(proj_name)
|
|
1169
1174
|
|
|
1170
1175
|
if not resolved_project_ids:
|
|
1171
|
-
|
|
1176
|
+
if ctx.obj.get("json"):
|
|
1177
|
+
click.echo(
|
|
1178
|
+
json_dumps(
|
|
1179
|
+
{"error": "NotFoundError", "message": "No matching projects found."}
|
|
1180
|
+
)
|
|
1181
|
+
)
|
|
1182
|
+
else:
|
|
1183
|
+
console.print("[yellow]No matching projects found.[/yellow]")
|
|
1172
1184
|
return
|
|
1173
1185
|
|
|
1174
1186
|
stats = client.get_run_stats(project_ids=resolved_project_ids)
|
|
@@ -1206,8 +1218,11 @@ def open_run(ctx, run_id):
|
|
|
1206
1218
|
# The SDK also has a way to get the URL but it might require project name.
|
|
1207
1219
|
url = f"https://smith.langchain.com/r/{run_id}"
|
|
1208
1220
|
|
|
1209
|
-
|
|
1210
|
-
|
|
1221
|
+
if ctx.obj.get("json"):
|
|
1222
|
+
click.echo(json_dumps({"run_id": run_id, "url": url}))
|
|
1223
|
+
else:
|
|
1224
|
+
click.echo(f"Opening run {run_id} in browser...")
|
|
1225
|
+
click.echo(f"URL: {url}")
|
|
1211
1226
|
webbrowser.open(url)
|
|
1212
1227
|
|
|
1213
1228
|
|
|
@@ -1447,7 +1462,6 @@ def search_runs(
|
|
|
1447
1462
|
roots=roots, # Pass through --roots flag
|
|
1448
1463
|
trace_filter=None,
|
|
1449
1464
|
tree_filter=None,
|
|
1450
|
-
order_by="-start_time",
|
|
1451
1465
|
reference_example_id=None,
|
|
1452
1466
|
tag=(),
|
|
1453
1467
|
name_pattern=None,
|
|
@@ -1664,7 +1678,6 @@ def sample_runs(
|
|
|
1664
1678
|
project_query=pq,
|
|
1665
1679
|
limit=sample_limit,
|
|
1666
1680
|
filter=combined_filter,
|
|
1667
|
-
order_by="-start_time",
|
|
1668
1681
|
console=console,
|
|
1669
1682
|
)
|
|
1670
1683
|
stratum_runs = result.items[:sample_limit]
|
|
@@ -1713,7 +1726,6 @@ def sample_runs(
|
|
|
1713
1726
|
project_query=pq,
|
|
1714
1727
|
limit=samples_per_stratum,
|
|
1715
1728
|
filter=combined_filter,
|
|
1716
|
-
order_by="-start_time",
|
|
1717
1729
|
console=console,
|
|
1718
1730
|
)
|
|
1719
1731
|
stratum_runs = result.items[:samples_per_stratum]
|
|
@@ -1954,7 +1966,6 @@ def analyze_runs(
|
|
|
1954
1966
|
project_query=pq,
|
|
1955
1967
|
filter=combined_filter,
|
|
1956
1968
|
limit=None,
|
|
1957
|
-
order_by="-start_time",
|
|
1958
1969
|
console=console,
|
|
1959
1970
|
)
|
|
1960
1971
|
all_runs = result.items
|
|
@@ -1973,7 +1984,6 @@ def analyze_runs(
|
|
|
1973
1984
|
**proj_kwargs,
|
|
1974
1985
|
filter=combined_filter,
|
|
1975
1986
|
limit=None, # SDK paginates automatically
|
|
1976
|
-
order_by="-start_time",
|
|
1977
1987
|
select=list(select_fields) if select_fields else None,
|
|
1978
1988
|
)
|
|
1979
1989
|
|
|
@@ -2142,7 +2152,6 @@ def _fetch_runs_for_discovery(
|
|
|
2142
2152
|
_fetch_runs,
|
|
2143
2153
|
project_query=pq,
|
|
2144
2154
|
limit=sample_size,
|
|
2145
|
-
order_by="-start_time",
|
|
2146
2155
|
select=select,
|
|
2147
2156
|
filter=combined_filter,
|
|
2148
2157
|
console=console,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import sys
|
|
2
2
|
import json as json_lib
|
|
3
3
|
import os
|
|
4
|
+
from typing import Any
|
|
4
5
|
import click
|
|
5
6
|
from rich.console import Console
|
|
6
7
|
from dotenv import load_dotenv
|
|
@@ -147,8 +148,40 @@ class LangSmithCLIGroup(click.Group):
|
|
|
147
148
|
sys.exit(1)
|
|
148
149
|
|
|
149
150
|
else:
|
|
150
|
-
#
|
|
151
|
-
|
|
151
|
+
# Non-LangSmith error (Click exceptions, Python exceptions, etc.)
|
|
152
|
+
from langsmith_cli.utils import CLIFetchError
|
|
153
|
+
|
|
154
|
+
if json_mode:
|
|
155
|
+
# In JSON mode, ALWAYS output structured JSON to stdout.
|
|
156
|
+
# Empty stdout breaks piped JSON parsing (json.loads fails).
|
|
157
|
+
if isinstance(e, CLIFetchError):
|
|
158
|
+
# Structured error with failure details and suggestions
|
|
159
|
+
error_data: dict[str, Any] = {
|
|
160
|
+
"error": "FetchError",
|
|
161
|
+
"message": e.format_message(),
|
|
162
|
+
"failed_sources": [
|
|
163
|
+
{"name": n, "error": err} for n, err in e.failed_sources
|
|
164
|
+
],
|
|
165
|
+
"suggestions": e.suggestions,
|
|
166
|
+
}
|
|
167
|
+
exit_code = e.exit_code
|
|
168
|
+
elif isinstance(e, click.ClickException):
|
|
169
|
+
error_data = {
|
|
170
|
+
"error": type(e).__name__,
|
|
171
|
+
"message": e.format_message(),
|
|
172
|
+
}
|
|
173
|
+
exit_code = e.exit_code
|
|
174
|
+
else:
|
|
175
|
+
error_data = {
|
|
176
|
+
"error": type(e).__name__,
|
|
177
|
+
"message": str(e),
|
|
178
|
+
}
|
|
179
|
+
exit_code = 1
|
|
180
|
+
click.echo(json_lib.dumps(error_data))
|
|
181
|
+
sys.exit(exit_code)
|
|
182
|
+
else:
|
|
183
|
+
# In human mode, re-raise for Click's default formatting
|
|
184
|
+
raise
|
|
152
185
|
finally:
|
|
153
186
|
# Flush stdout to prevent data loss when piping to other processes
|
|
154
187
|
# This fixes race conditions where buffered output may not reach the pipe
|