dremio-cli 1.5.0__tar.gz → 1.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dremio_cli-1.5.0/dremio_cli.egg-info → dremio_cli-1.7.0}/PKG-INFO +13 -2
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/README.md +10 -1
- dremio_cli-1.7.0/docs/dac.md +60 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/jobs.md +13 -0
- dremio_cli-1.7.0/docs/lineage.md +25 -0
- dremio_cli-1.7.0/docs/profiles.md +139 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/sql.md +3 -1
- dremio_cli-1.7.0/docs/tui.md +44 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/cli.py +6 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/auth.py +6 -1
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/cloud.py +6 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/software.py +10 -5
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/job.py +82 -0
- dremio_cli-1.7.0/dremio_cli/commands/lineage.py +88 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/sql.py +20 -1
- dremio_cli-1.7.0/dremio_cli/commands/sync.py +51 -0
- dremio_cli-1.7.0/dremio_cli/commands/ui.py +20 -0
- dremio_cli-1.7.0/dremio_cli/dac/__init__.py +3 -0
- dremio_cli-1.7.0/dremio_cli/dac/config.py +42 -0
- dremio_cli-1.7.0/dremio_cli/dac/diff.py +26 -0
- dremio_cli-1.7.0/dremio_cli/dac/state.py +84 -0
- dremio_cli-1.7.0/dremio_cli/dac/sync.py +344 -0
- dremio_cli-1.7.0/dremio_cli/tui/catalog/__init__.py +1 -0
- dremio_cli-1.7.0/dremio_cli/tui/catalog/app.py +34 -0
- dremio_cli-1.7.0/dremio_cli/tui/catalog/screens.py +117 -0
- dremio_cli-1.7.0/dremio_cli/tui/catalog/widgets.py +180 -0
- dremio_cli-1.7.0/dremio_cli/utils/console.py +4 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0/dremio_cli.egg-info}/PKG-INFO +13 -2
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli.egg-info/SOURCES.txt +15 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli.egg-info/requires.txt +2 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/pyproject.toml +3 -1
- dremio_cli-1.5.0/docs/profiles.md +0 -378
- dremio_cli-1.5.0/dremio_cli/commands/lineage.py +0 -9
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/LICENSE +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/MANIFEST.in +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/completions/dremio-completion.bash +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/completions/dremio-completion.zsh +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/README.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/advanced-features.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/catalog.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/completion.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/favorites.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/grants.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/history.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/init.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/installation.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/monitor.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/quickstart.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/reflections.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/repl.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/roles.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/scripts.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/sources.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/spaces-folders.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/tables.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/tags-wiki.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/users.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/views.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/__main__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/base.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/factory.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/catalog.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/completion.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/favorite.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/folder.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/grant.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/history.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/init.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/monitor.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/profile.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/reflection.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/repl.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/role.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/script.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/source.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/space.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/table.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/tag.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/udf.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/user.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/view.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/wiki.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/config.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/env.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/formatters/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/formatters/json.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/formatters/table.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/formatters/yaml.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/history.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/utils/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/utils/exceptions.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/utils/helpers.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/utils/validators.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli.egg-info/dependency_links.txt +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli.egg-info/entry_points.txt +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli.egg-info/top_level.txt +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/setup.cfg +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/setup.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/conftest.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_config.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_grant_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_history_favorite.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_integration.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_job_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_multi_sql_execution.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_reflection_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_role_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_script_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_source_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_space_folder_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_sql_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_table_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_tag_wiki_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_user_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_view_commands.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dremio-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.7.0
|
|
4
4
|
Summary: A comprehensive command-line interface for Dremio Cloud and Dremio Software
|
|
5
5
|
Home-page: https://github.com/your-org/dremio-cli
|
|
6
6
|
Author: Your Name
|
|
@@ -38,6 +38,8 @@ Requires-Dist: python-dotenv>=1.0.0
|
|
|
38
38
|
Requires-Dist: pyyaml>=6.0.0
|
|
39
39
|
Requires-Dist: prompt_toolkit>=3.0.0
|
|
40
40
|
Requires-Dist: textual>=0.40.0
|
|
41
|
+
Requires-Dist: pandas>=2.0.0
|
|
42
|
+
Requires-Dist: pyarrow>=12.0.0
|
|
41
43
|
Provides-Extra: dev
|
|
42
44
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
43
45
|
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
|
@@ -88,7 +90,10 @@ dremio view create --path "Analytics.summary" \
|
|
|
88
90
|
|
|
89
91
|
## 📚 Documentation
|
|
90
92
|
|
|
91
|
-
**[Complete Documentation →](docs
|
|
93
|
+
**[Complete Documentation (GitHub) →](https://github.com/developer-advocacy-dremio/dremio-python-cli/tree/main/dremio-cli/docs)**
|
|
94
|
+
|
|
95
|
+
*(Use the link above if browsing on PyPI)*
|
|
96
|
+
|
|
92
97
|
|
|
93
98
|
### Core Guides
|
|
94
99
|
|
|
@@ -118,6 +123,12 @@ dremio view create --path "Analytics.summary" \
|
|
|
118
123
|
|
|
119
124
|
- **[Favorites](docs/favorites.md)** - Save frequently used queries
|
|
120
125
|
- **[History](docs/history.md)** - View and re-run commands
|
|
126
|
+
- **[REPL](docs/repl.md)** - Interactive SQL Shell (Enhanced)
|
|
127
|
+
- **[Monitor](docs/monitor.md)** - Real-time System Monitor
|
|
128
|
+
- **[Catalog Explorer](docs/tui.md)** - Interactive Catalog TUI
|
|
129
|
+
- **[Init](docs/init.md)** - Interactive Configuration Wizard
|
|
130
|
+
- **[Completion](docs/completion.md)** - Shell Completion Scripts
|
|
131
|
+
- **[Lineage](docs/lineage.md)** - Visualize dataset dependencies
|
|
121
132
|
|
|
122
133
|
## 🎯 Key Features
|
|
123
134
|
|
|
@@ -37,7 +37,10 @@ dremio view create --path "Analytics.summary" \
|
|
|
37
37
|
|
|
38
38
|
## 📚 Documentation
|
|
39
39
|
|
|
40
|
-
**[Complete Documentation →](docs
|
|
40
|
+
**[Complete Documentation (GitHub) →](https://github.com/developer-advocacy-dremio/dremio-python-cli/tree/main/dremio-cli/docs)**
|
|
41
|
+
|
|
42
|
+
*(Use the link above if browsing on PyPI)*
|
|
43
|
+
|
|
41
44
|
|
|
42
45
|
### Core Guides
|
|
43
46
|
|
|
@@ -67,6 +70,12 @@ dremio view create --path "Analytics.summary" \
|
|
|
67
70
|
|
|
68
71
|
- **[Favorites](docs/favorites.md)** - Save frequently used queries
|
|
69
72
|
- **[History](docs/history.md)** - View and re-run commands
|
|
73
|
+
- **[REPL](docs/repl.md)** - Interactive SQL Shell (Enhanced)
|
|
74
|
+
- **[Monitor](docs/monitor.md)** - Real-time System Monitor
|
|
75
|
+
- **[Catalog Explorer](docs/tui.md)** - Interactive Catalog TUI
|
|
76
|
+
- **[Init](docs/init.md)** - Interactive Configuration Wizard
|
|
77
|
+
- **[Completion](docs/completion.md)** - Shell Completion Scripts
|
|
78
|
+
- **[Lineage](docs/lineage.md)** - Visualize dataset dependencies
|
|
70
79
|
|
|
71
80
|
## 🎯 Key Features
|
|
72
81
|
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Dremio-as-Code (GitOps)
|
|
2
|
+
|
|
3
|
+
Dremio-as-Code (DAC) allows you to manage your Dremio catalog (Spaces, Folders, Views) using local files, enabling GitOps workflows.
|
|
4
|
+
|
|
5
|
+
## Quick Start
|
|
6
|
+
|
|
7
|
+
### 1. Initialize Configuration
|
|
8
|
+
Create a `dremio.yaml` file in your project root:
|
|
9
|
+
|
|
10
|
+
```yaml
|
|
11
|
+
version: "1.0"
|
|
12
|
+
scope:
|
|
13
|
+
path: "dremio-catalog.finance" # The Dremio path to sync
|
|
14
|
+
type: "SPACE" # SPACE or ICEBERGCATALOG
|
|
15
|
+
ignore:
|
|
16
|
+
- "*.tmp"
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
### 2. Pull State
|
|
20
|
+
Capture the current state of your Dremio space into local files.
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
dremio sync pull
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Returns a directory structure mirroring Dremio:
|
|
27
|
+
- `finance/`
|
|
28
|
+
- `monthly_report.sql`
|
|
29
|
+
- `monthly_report.yaml`
|
|
30
|
+
|
|
31
|
+
### 3. Make Changes
|
|
32
|
+
Edit a view's SQL locally or create a new file pair (`.sql` and `.yaml`).
|
|
33
|
+
|
|
34
|
+
**new_view.sql**:
|
|
35
|
+
```sql
|
|
36
|
+
SELECT * FROM parent.table
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
**new_view.yaml**:
|
|
40
|
+
```yaml
|
|
41
|
+
type: VIEW
|
|
42
|
+
path: ["dremio-catalog", "finance", "new_view"]
|
|
43
|
+
sql_file: "new_view.sql"
|
|
44
|
+
context: []
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
### 4. Push Changes
|
|
48
|
+
Apply your local changes back to Dremio.
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
dremio sync push
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Concepts
|
|
55
|
+
|
|
56
|
+
- **Scope**: Limits the sync to a specific subtree to support multi-team environments.
|
|
57
|
+
- **State File**: `.dremio_state.json` tracks the last known state to enable efficient updates and deletes. **Do not edit this file manually.**
|
|
58
|
+
- **File Format**:
|
|
59
|
+
- **Context**: List of path parts to set query context.
|
|
60
|
+
- **Reflections**: (Coming Soon) Defined in YAML.
|
|
@@ -60,6 +60,19 @@ dremio --output json job get 16b2c9cd-a920-952b-b162-2280c9059d00
|
|
|
60
60
|
dremio --verbose job get 16b2c9cd-a920-952b-b162-2280c9059d00
|
|
61
61
|
```
|
|
62
62
|
|
|
63
|
+
### Analyze Job
|
|
64
|
+
|
|
65
|
+
Analyze the performance of a job to identify bottlenecks.
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
dremio job analyze <JOB_ID>
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
**Output:**
|
|
72
|
+
- Job duration and state
|
|
73
|
+
- Data reduction ratio (input vs output records)
|
|
74
|
+
- Insights on performance metrics
|
|
75
|
+
|
|
63
76
|
### Get Job Results
|
|
64
77
|
|
|
65
78
|
Retrieve the results of a completed job.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Lineage Visualization
|
|
2
|
+
|
|
3
|
+
Visualize the dependencies between datasets.
|
|
4
|
+
|
|
5
|
+
## Commands
|
|
6
|
+
|
|
7
|
+
### Show Lineage
|
|
8
|
+
|
|
9
|
+
Show the upstream parents of a dataset.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
dremio lineage show <CATALOG_ID> [OPTIONS]
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
**Options:**
|
|
16
|
+
- `--format [tree|json|mermaid]` - Output format (Default: tree)
|
|
17
|
+
|
|
18
|
+
**Examples:**
|
|
19
|
+
```bash
|
|
20
|
+
# Tree view (Terminal)
|
|
21
|
+
dremio lineage show dremio-catalog.space.view
|
|
22
|
+
|
|
23
|
+
# Mermaid Graph (for markdown)
|
|
24
|
+
dremio lineage show dremio-catalog.space.view --format mermaid
|
|
25
|
+
```
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# Profile Management Guide
|
|
2
|
+
|
|
3
|
+
This guide covers how to create and manage Dremio CLI profiles using both YAML configuration files and environment variables.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
Profiles store connection information for Dremio instances. The CLI supports two methods:
|
|
8
|
+
|
|
9
|
+
1. **YAML Configuration** - Stored in `~/.dremio/profiles.yaml` (RECOMMENDED for local use)
|
|
10
|
+
2. **Environment Variables** - Loaded from `.env` file or shell environment (RECOMMENDED for CI/CD)
|
|
11
|
+
|
|
12
|
+
Environment variables take precedence over YAML profiles.
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
## 🚀 Profile Values Guide
|
|
17
|
+
|
|
18
|
+
### 1. Base URL
|
|
19
|
+
The URL to your Dremio instance's API.
|
|
20
|
+
|
|
21
|
+
| Platform | Format | Example | Note |
|
|
22
|
+
|----------|--------|---------|------|
|
|
23
|
+
| **Dremio Cloud** | `https://api.dremio.cloud/v0` | `https://api.dremio.cloud/v0` | Used for US control plane |
|
|
24
|
+
| **Dremio Cloud (EU)** | `https://api.dremio.eu/v0` | `https://api.dremio.eu/v0` | Used for EU control plane |
|
|
25
|
+
| **Dremio Software** | `http(s)://<host>:<port>` | `https://dremio.company.com` | **Smart URL**: The CLI automatically appends `/api/v3` if you omit it. |
|
|
26
|
+
| **Local Software** | `http://localhost:9047` | `http://localhost:9047` | Defaults for local Docker/install |
|
|
27
|
+
|
|
28
|
+
> **Note:** For Dremio Software, you can provide `https://dremio.company.com` OR `https://dremio.company.com/api/v3`. The CLI handles both correctly.
|
|
29
|
+
|
|
30
|
+
### 2. Authentication
|
|
31
|
+
How you log in to Dremio.
|
|
32
|
+
|
|
33
|
+
| Type | Platform | Description |
|
|
34
|
+
|------|----------|-------------|
|
|
35
|
+
| **PAT** (Token) | Cloud & Software | **Recommended**. Personal Access Token generated in User Settings. |
|
|
36
|
+
| **Services Account** | Cloud Only | Treats Client/Secret as a PAT for automation. |
|
|
37
|
+
| **Username/Password** | Software Only | Traditional login. **Less secure** than PAT. |
|
|
38
|
+
|
|
39
|
+
### 3. Project ID (Cloud Only)
|
|
40
|
+
Can be found in the URL of your Dremio Cloud project.
|
|
41
|
+
- URL: `https://app.dremio.cloud/projectId/12345-abcde.../home`
|
|
42
|
+
- Project ID: `12345-abcde...`
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## YAML Configuration
|
|
47
|
+
|
|
48
|
+
**Location**: `~/.dremio/profiles.yaml`
|
|
49
|
+
|
|
50
|
+
### Examples
|
|
51
|
+
|
|
52
|
+
**Dremio Cloud (US)**
|
|
53
|
+
```yaml
|
|
54
|
+
profiles:
|
|
55
|
+
cloud-prod:
|
|
56
|
+
type: cloud
|
|
57
|
+
base_url: https://api.dremio.cloud/v0
|
|
58
|
+
project_id: 788baab4-3c3b-42da-9f1d-5cc6dc03147d
|
|
59
|
+
auth:
|
|
60
|
+
type: pat
|
|
61
|
+
token: your-personal-access-token
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
**Dremio Software (Corporate)**
|
|
65
|
+
```yaml
|
|
66
|
+
profiles:
|
|
67
|
+
corp-dremio:
|
|
68
|
+
type: software
|
|
69
|
+
base_url: https://dremio.corp.com # CLI will add /api/v3 automatically
|
|
70
|
+
auth:
|
|
71
|
+
type: pat
|
|
72
|
+
token: your-personal-access-token
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
**Dremio Software (Local/Docker)**
|
|
76
|
+
```yaml
|
|
77
|
+
profiles:
|
|
78
|
+
local:
|
|
79
|
+
type: software
|
|
80
|
+
base_url: http://localhost:9047 # Default port
|
|
81
|
+
auth:
|
|
82
|
+
type: username_password
|
|
83
|
+
username: admin
|
|
84
|
+
password: password123
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### CLI Commands
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
# Interactve Wizard (Best for beginners)
|
|
91
|
+
dremio init
|
|
92
|
+
|
|
93
|
+
# Create manually
|
|
94
|
+
dremio profile create --name prod --type cloud ...
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
## Environment Variable Configuration
|
|
100
|
+
|
|
101
|
+
Ideal for CI/CD pipelines or Docker containers.
|
|
102
|
+
|
|
103
|
+
### Pattern
|
|
104
|
+
`DREMIO_{PROFILE_NAME}_KEY=VALUE`
|
|
105
|
+
|
|
106
|
+
### Example `.env` File
|
|
107
|
+
```bash
|
|
108
|
+
# Cloud Profile (Name: 'CLOUD')
|
|
109
|
+
DREMIO_CLOUD_TYPE=cloud
|
|
110
|
+
DREMIO_CLOUD_BASE_URL=https://api.dremio.cloud/v0
|
|
111
|
+
DREMIO_CLOUD_PROJECTID=788baab4-3c3b-42da-9f1d-5cc6dc03147d
|
|
112
|
+
DREMIO_CLOUD_TOKEN=s3JcLOqFTR...
|
|
113
|
+
|
|
114
|
+
# Software Profile (Name: 'PROD')
|
|
115
|
+
DREMIO_PROD_TYPE=software
|
|
116
|
+
DREMIO_PROD_BASE_URL=https://dremio.corp.com
|
|
117
|
+
DREMIO_PROD_TOKEN=Q/ToosxORA...
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
### Usage
|
|
121
|
+
```bash
|
|
122
|
+
# Authenticates using DREMIO_PROD_* variables
|
|
123
|
+
dremio --profile prod catalog list
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
---
|
|
127
|
+
|
|
128
|
+
## Profile Management
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
# List all profiles
|
|
132
|
+
dremio profile list
|
|
133
|
+
|
|
134
|
+
# Show active profile details
|
|
135
|
+
dremio profile current
|
|
136
|
+
|
|
137
|
+
# Set default profile (so you don't need --profile flag)
|
|
138
|
+
dremio profile set-default cloud-prod
|
|
139
|
+
```
|
|
@@ -20,7 +20,9 @@ dremio sql execute --file <FILE> [OPTIONS]
|
|
|
20
20
|
- `--file PATH` - Execute SQL from file
|
|
21
21
|
- `--context TEXT` - Query context (comma-separated path)
|
|
22
22
|
- `--async` - Execute asynchronously (return job ID immediately). Default behavior waits for results.
|
|
23
|
-
- `--output-file PATH` - Save results to file
|
|
23
|
+
- `--output-file PATH` - Save results to file (supports .json, .yaml, .csv, .parquet)
|
|
24
|
+
|
|
25
|
+
**Note:** `.csv` and `.parquet` export require `pandas` and `pyarrow` installed.
|
|
24
26
|
|
|
25
27
|
**Examples:**
|
|
26
28
|
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Catalog Explorer (TUI)
|
|
2
|
+
|
|
3
|
+
The Dremio CLI includes an interactive Terminal User Interface (TUI) for exploring your Dremio catalog, viewing schemas, and previewing data without leaving the command line.
|
|
4
|
+
|
|
5
|
+
## Usage
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
dremio ui catalog
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Or using a specific profile:
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
dremio --profile prod ui catalog
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Features
|
|
18
|
+
|
|
19
|
+
### 1. Interactive Navigation
|
|
20
|
+
- **Tree View**: Navigate seamlessly through Spaces, Sources, Folders, and your Home directory.
|
|
21
|
+
- **Async Loading**: Large catalogs load quickly as you expand nodes.
|
|
22
|
+
|
|
23
|
+
### 2. Dataset Details
|
|
24
|
+
Select any dataset (View/Table) to see:
|
|
25
|
+
- **Info**: Metadata, ID, and Type.
|
|
26
|
+
- **Schema**: Column names and types.
|
|
27
|
+
- **SQL**: The underlying SQL definition (for Views).
|
|
28
|
+
|
|
29
|
+
### 3. Data Preview
|
|
30
|
+
- Switch to the **Preview** tab to instantly run a query (`SELECT * ... LIMIT 20`) and view actual data samples in a table.
|
|
31
|
+
|
|
32
|
+
## Controls
|
|
33
|
+
|
|
34
|
+
| Key | Action |
|
|
35
|
+
|-----|--------|
|
|
36
|
+
| `↑` / `↓` | Navigate Tree |
|
|
37
|
+
| `Space` | Expand/Collapse Folder |
|
|
38
|
+
| `Enter` | Select Item |
|
|
39
|
+
| `Click` | Select tabs (Info, Schema, Preview) |
|
|
40
|
+
| `q` | Quit |
|
|
41
|
+
|
|
42
|
+
## Requirements
|
|
43
|
+
- Dremio Software or Dremio Cloud
|
|
44
|
+
- Terminal with UTF-8 support
|
|
@@ -30,6 +30,9 @@ from dremio_cli.commands import (
|
|
|
30
30
|
repl,
|
|
31
31
|
monitor,
|
|
32
32
|
completion,
|
|
33
|
+
lineage,
|
|
34
|
+
sync,
|
|
35
|
+
ui,
|
|
33
36
|
)
|
|
34
37
|
|
|
35
38
|
|
|
@@ -102,6 +105,9 @@ cli.add_command(init.init_command)
|
|
|
102
105
|
cli.add_command(repl.repl_command)
|
|
103
106
|
cli.add_command(monitor.monitor_command)
|
|
104
107
|
cli.add_command(completion.completion_command)
|
|
108
|
+
cli.add_command(lineage.lineage)
|
|
109
|
+
cli.add_command(sync.sync)
|
|
110
|
+
cli.add_command(ui.ui)
|
|
105
111
|
|
|
106
112
|
|
|
107
113
|
|
|
@@ -25,7 +25,12 @@ def authenticate_with_username_password(
|
|
|
25
25
|
AuthenticationError: If authentication fails
|
|
26
26
|
"""
|
|
27
27
|
# Use v2 API for login
|
|
28
|
-
|
|
28
|
+
# Normalize base_url to get root
|
|
29
|
+
root_url = base_url.rstrip("/")
|
|
30
|
+
if root_url.endswith("/api/v3"):
|
|
31
|
+
root_url = root_url[:-7] # remove /api/v3
|
|
32
|
+
|
|
33
|
+
login_url = f"{root_url}/apiv2/login"
|
|
29
34
|
|
|
30
35
|
try:
|
|
31
36
|
response = requests.post(
|
|
@@ -363,3 +363,9 @@ class CloudClient(BaseClient):
|
|
|
363
363
|
"""Delete a script."""
|
|
364
364
|
return self.delete(self._project_endpoint(f"scripts/{script_id}"))
|
|
365
365
|
|
|
366
|
+
# Lineage operations
|
|
367
|
+
def get_catalog_graph(self, catalog_id: str) -> Dict[str, Any]:
|
|
368
|
+
"""Get lineage graph for a catalog item."""
|
|
369
|
+
# Cloud API usually follows similar pattern or project endpoint
|
|
370
|
+
return self.get(self._project_endpoint(f"catalog/{catalog_id}/graph"))
|
|
371
|
+
|
|
@@ -21,6 +21,11 @@ class SoftwareClient(BaseClient):
|
|
|
21
21
|
token: Authentication token
|
|
22
22
|
timeout: Request timeout in seconds
|
|
23
23
|
"""
|
|
24
|
+
# Auto-append /api/v3 if missing
|
|
25
|
+
base_url = base_url.rstrip("/")
|
|
26
|
+
if not base_url.endswith("/api/v3"):
|
|
27
|
+
base_url = f"{base_url}/api/v3"
|
|
28
|
+
|
|
24
29
|
super().__init__(base_url, token, timeout)
|
|
25
30
|
|
|
26
31
|
# Catalog operations
|
|
@@ -609,10 +614,10 @@ class SoftwareClient(BaseClient):
|
|
|
609
614
|
return self.put(f"reflection/{reflection_id}", data=reflection_data)
|
|
610
615
|
|
|
611
616
|
def delete_reflection(self, reflection_id: str) -> None:
|
|
612
|
-
"""Delete a reflection.
|
|
613
|
-
|
|
614
|
-
Args:
|
|
615
|
-
reflection_id: Reflection ID
|
|
616
|
-
"""
|
|
617
|
+
"""Delete a reflection."""
|
|
617
618
|
return self.delete(f"reflection/{reflection_id}")
|
|
618
619
|
|
|
620
|
+
# Lineage operations
|
|
621
|
+
def get_catalog_graph(self, catalog_id: str) -> Dict[str, Any]:
|
|
622
|
+
"""Get lineage graph for a catalog item."""
|
|
623
|
+
return self.get(f"catalog/{catalog_id}/graph")
|
|
@@ -319,3 +319,85 @@ def get_job_reflections(ctx, job_id: str) -> None:
|
|
|
319
319
|
import traceback
|
|
320
320
|
traceback.print_exc()
|
|
321
321
|
raise click.Abort()
|
|
322
|
+
@job.command("analyze")
|
|
323
|
+
@click.argument("job_id")
|
|
324
|
+
@click.pass_context
|
|
325
|
+
def analyze_job(ctx, job_id: str) -> None:
|
|
326
|
+
"""Analyze job performance.
|
|
327
|
+
|
|
328
|
+
Examples:
|
|
329
|
+
dremio job analyze <job-id>
|
|
330
|
+
"""
|
|
331
|
+
try:
|
|
332
|
+
manager = ProfileManager()
|
|
333
|
+
profile_name = ctx.obj.profile_name
|
|
334
|
+
profile = manager.get_profile(profile_name)
|
|
335
|
+
|
|
336
|
+
if not profile:
|
|
337
|
+
console.print(f"[red]Profile '{profile_name}' not found[/red]")
|
|
338
|
+
return
|
|
339
|
+
|
|
340
|
+
client = create_client(profile)
|
|
341
|
+
|
|
342
|
+
with console.status(f"[bold green]Analyzing job {job_id}..."):
|
|
343
|
+
job = client.get_job(job_id)
|
|
344
|
+
|
|
345
|
+
# Basic info
|
|
346
|
+
state = job.get("jobState")
|
|
347
|
+
start = job.get("startTime")
|
|
348
|
+
end = job.get("endTime")
|
|
349
|
+
|
|
350
|
+
# Stats (structure varies between Cloud/Software versions)
|
|
351
|
+
# Safe access to stats
|
|
352
|
+
stats = job.get("stats", {})
|
|
353
|
+
|
|
354
|
+
# Provide insights
|
|
355
|
+
from rich.panel import Panel
|
|
356
|
+
from rich.columns import Columns
|
|
357
|
+
|
|
358
|
+
# Duration
|
|
359
|
+
duration = "N/A"
|
|
360
|
+
if start and end:
|
|
361
|
+
# Basic duration calc if not in stats (assuming epoch ms)
|
|
362
|
+
try:
|
|
363
|
+
diff = (int(end) - int(start)) / 1000.0
|
|
364
|
+
duration = f"{diff:.2f}s"
|
|
365
|
+
except:
|
|
366
|
+
pass
|
|
367
|
+
|
|
368
|
+
# Input/Output
|
|
369
|
+
input_bytes = stats.get("inputBytes", 0)
|
|
370
|
+
output_bytes = stats.get("outputBytes", 0)
|
|
371
|
+
|
|
372
|
+
# Simple insights
|
|
373
|
+
insights = []
|
|
374
|
+
if state != "COMPLETED":
|
|
375
|
+
insights.append(f"[red]Job did not complete ({state})[/red]")
|
|
376
|
+
|
|
377
|
+
input_records = stats.get("inputRecords", 0)
|
|
378
|
+
output_records = stats.get("outputRecords", 0)
|
|
379
|
+
if input_records > 0 and output_records > 0:
|
|
380
|
+
reduction = (1 - (output_records / input_records)) * 100
|
|
381
|
+
insights.append(f"[green]Data Reduction: {reduction:.1f}%[/green]")
|
|
382
|
+
|
|
383
|
+
# Display
|
|
384
|
+
console.print(Panel(f"""
|
|
385
|
+
[bold]Job ID[/bold]: {job_id}
|
|
386
|
+
[bold]State[/bold]: {state}
|
|
387
|
+
[bold]Duration[/bold]: {duration}
|
|
388
|
+
[bold]User[/bold]: {job.get('user', 'N/A')}
|
|
389
|
+
""", title="Job Summary", border_style="cyan"))
|
|
390
|
+
|
|
391
|
+
if insights:
|
|
392
|
+
console.print(Panel("\n".join(insights), title="Insights", border_style="yellow"))
|
|
393
|
+
|
|
394
|
+
# Raw stats table
|
|
395
|
+
from rich.table import Table
|
|
396
|
+
t = Table(title="Metrics", show_header=False)
|
|
397
|
+
t.add_row("Input Records", str(input_records))
|
|
398
|
+
t.add_row("Output Records", str(output_records))
|
|
399
|
+
t.add_row("Memory Used", f"{stats.get('memoryAllocation', 0)} bytes")
|
|
400
|
+
console.print(t)
|
|
401
|
+
|
|
402
|
+
except Exception as e:
|
|
403
|
+
console.print(f"[red]Analysis failed: {e}[/red]")
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Lineage management commands."""
|
|
2
|
+
|
|
3
|
+
import click
|
|
4
|
+
import json
|
|
5
|
+
from rich.console import Console
|
|
6
|
+
from rich.tree import Tree
|
|
7
|
+
|
|
8
|
+
from dremio_cli.config import ProfileManager
|
|
9
|
+
from dremio_cli.client.factory import create_client
|
|
10
|
+
from dremio_cli.formatters.json import format_as_json
|
|
11
|
+
|
|
12
|
+
console = Console()
|
|
13
|
+
|
|
14
|
+
@click.group()
|
|
15
|
+
def lineage():
|
|
16
|
+
"""Lineage visualization operations."""
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
@lineage.command("show")
|
|
20
|
+
@click.argument("catalog_id")
|
|
21
|
+
@click.option("--format", type=click.Choice(["tree", "json", "mermaid"], case_sensitive=False), default="tree", help="Output format")
|
|
22
|
+
@click.pass_context
|
|
23
|
+
def show_lineage(ctx, catalog_id: str, format: str):
|
|
24
|
+
"""Show lineage graph for a dataset.
|
|
25
|
+
|
|
26
|
+
Examples:
|
|
27
|
+
dremio lineage show <catalog-id>
|
|
28
|
+
dremio lineage show <catalog-id> --format mermaid
|
|
29
|
+
"""
|
|
30
|
+
try:
|
|
31
|
+
manager = ProfileManager()
|
|
32
|
+
profile_name = ctx.obj.profile_name
|
|
33
|
+
profile = manager.get_profile(profile_name)
|
|
34
|
+
|
|
35
|
+
if not profile:
|
|
36
|
+
console.print(f"[red]Profile '{profile_name}' not found[/red]")
|
|
37
|
+
return
|
|
38
|
+
|
|
39
|
+
client = create_client(profile)
|
|
40
|
+
|
|
41
|
+
try:
|
|
42
|
+
graph = client.get_catalog_graph(catalog_id)
|
|
43
|
+
except Exception as e:
|
|
44
|
+
console.print(f"[red]Error fetching lineage: {e}[/red]")
|
|
45
|
+
# Fallback or specific error handling (e.g. 404)
|
|
46
|
+
return
|
|
47
|
+
|
|
48
|
+
if format == "json":
|
|
49
|
+
console.print(format_as_json(graph))
|
|
50
|
+
return
|
|
51
|
+
|
|
52
|
+
# Process graph for Tree/Mermaid
|
|
53
|
+
# Graph structure usually: { "parents": [...], "children": [...] }
|
|
54
|
+
# Or detailed nodes/edges depending on API version.
|
|
55
|
+
# Assuming Dremio V3 graph response structure:
|
|
56
|
+
# { "parents": [ { "id": "...", "path": [...] } ], "children": [] }
|
|
57
|
+
|
|
58
|
+
# We focus on parents (upstream lineage)
|
|
59
|
+
parents = graph.get("parents", [])
|
|
60
|
+
|
|
61
|
+
if format == "mermaid":
|
|
62
|
+
click.echo("graph TD")
|
|
63
|
+
# Current node
|
|
64
|
+
click.echo(f' root["{catalog_id}"]')
|
|
65
|
+
click.echo(" style root fill:#f9f,stroke:#333,stroke-width:2px")
|
|
66
|
+
|
|
67
|
+
for parent in parents:
|
|
68
|
+
p_path = ".".join(parent.get("path", []))
|
|
69
|
+
p_id = parent.get("id", p_path) # fallback if id missing
|
|
70
|
+
# Sanitize ID for mermaid
|
|
71
|
+
safe_pid = p_id.replace("-", "_").replace(".", "_")
|
|
72
|
+
|
|
73
|
+
click.echo(f' {safe_pid}["{p_path}"] --> root')
|
|
74
|
+
|
|
75
|
+
else: # tree
|
|
76
|
+
tree = Tree(f"[bold cyan]{catalog_id}[/bold cyan]")
|
|
77
|
+
if parents:
|
|
78
|
+
p_branch = tree.add("Parents")
|
|
79
|
+
for parent in parents:
|
|
80
|
+
p_path = ".".join(parent.get("path", []))
|
|
81
|
+
p_branch.add(f"[green]{p_path}[/green]")
|
|
82
|
+
else:
|
|
83
|
+
tree.add("[italic]No parents found[/italic]")
|
|
84
|
+
|
|
85
|
+
console.print(tree)
|
|
86
|
+
|
|
87
|
+
except Exception as e:
|
|
88
|
+
console.print(f"[red]Error: {e}[/red]")
|
|
@@ -148,7 +148,26 @@ def execute_sql(ctx, query: str, sql_file: str, context: str, async_mode: bool,
|
|
|
148
148
|
console.print("[yellow]Warning: --output-file with multiple queries will subscribe/overwite. Currently not fully supported for multi-statement.[/yellow]")
|
|
149
149
|
|
|
150
150
|
output_path = Path(output_file)
|
|
151
|
-
|
|
151
|
+
|
|
152
|
+
# Advanced Formats (Pandas required)
|
|
153
|
+
if output_path.suffix in ['.parquet', '.csv']:
|
|
154
|
+
try:
|
|
155
|
+
import pandas as pd
|
|
156
|
+
df = pd.DataFrame(rows)
|
|
157
|
+
|
|
158
|
+
if output_path.suffix == '.parquet':
|
|
159
|
+
df.to_parquet(output_path)
|
|
160
|
+
else: # csv
|
|
161
|
+
df.to_csv(output_path, index=False)
|
|
162
|
+
|
|
163
|
+
except ImportError:
|
|
164
|
+
console.print("[red]Error: 'pandas' and 'pyarrow' are required for Parquet/CSV export. Install them with pip.[/red]")
|
|
165
|
+
raise click.Abort()
|
|
166
|
+
except Exception as e:
|
|
167
|
+
console.print(f"[red]Export failed: {e}[/red]")
|
|
168
|
+
raise click.Abort()
|
|
169
|
+
|
|
170
|
+
elif output_path.suffix == '.json':
|
|
152
171
|
output_path.write_text(json.dumps(results, indent=2))
|
|
153
172
|
elif output_path.suffix in ['.yaml', '.yml']:
|
|
154
173
|
import yaml
|