dremio-cli 1.5.0__tar.gz → 1.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dremio_cli-1.5.0/dremio_cli.egg-info → dremio_cli-1.6.0}/PKG-INFO +12 -2
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/README.md +9 -1
- dremio_cli-1.6.0/docs/dac.md +60 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/jobs.md +13 -0
- dremio_cli-1.6.0/docs/lineage.md +25 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/sql.md +3 -1
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/cli.py +4 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/cloud.py +6 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/software.py +5 -5
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/job.py +82 -0
- dremio_cli-1.6.0/dremio_cli/commands/lineage.py +88 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/sql.py +20 -1
- dremio_cli-1.6.0/dremio_cli/commands/sync.py +51 -0
- dremio_cli-1.6.0/dremio_cli/dac/__init__.py +3 -0
- dremio_cli-1.6.0/dremio_cli/dac/config.py +42 -0
- dremio_cli-1.6.0/dremio_cli/dac/diff.py +26 -0
- dremio_cli-1.6.0/dremio_cli/dac/state.py +84 -0
- dremio_cli-1.6.0/dremio_cli/dac/sync.py +344 -0
- dremio_cli-1.6.0/dremio_cli/utils/console.py +4 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0/dremio_cli.egg-info}/PKG-INFO +12 -2
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli.egg-info/SOURCES.txt +9 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli.egg-info/requires.txt +2 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/pyproject.toml +3 -1
- dremio_cli-1.5.0/dremio_cli/commands/lineage.py +0 -9
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/LICENSE +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/MANIFEST.in +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/completions/dremio-completion.bash +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/completions/dremio-completion.zsh +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/README.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/advanced-features.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/catalog.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/completion.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/favorites.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/grants.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/history.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/init.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/installation.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/monitor.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/profiles.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/quickstart.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/reflections.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/repl.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/roles.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/scripts.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/sources.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/spaces-folders.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/tables.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/tags-wiki.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/users.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/views.md +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/__main__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/auth.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/base.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/factory.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/catalog.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/completion.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/favorite.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/folder.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/grant.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/history.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/init.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/monitor.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/profile.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/reflection.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/repl.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/role.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/script.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/source.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/space.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/table.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/tag.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/udf.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/user.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/view.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/wiki.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/config.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/env.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/formatters/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/formatters/json.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/formatters/table.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/formatters/yaml.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/history.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/utils/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/utils/exceptions.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/utils/helpers.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/utils/validators.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli.egg-info/dependency_links.txt +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli.egg-info/entry_points.txt +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli.egg-info/top_level.txt +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/setup.cfg +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/setup.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/__init__.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/conftest.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_config.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_grant_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_history_favorite.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_integration.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_job_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_multi_sql_execution.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_reflection_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_role_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_script_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_source_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_space_folder_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_sql_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_table_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_tag_wiki_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_user_commands.py +0 -0
- {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_view_commands.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dremio-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.6.0
|
|
4
4
|
Summary: A comprehensive command-line interface for Dremio Cloud and Dremio Software
|
|
5
5
|
Home-page: https://github.com/your-org/dremio-cli
|
|
6
6
|
Author: Your Name
|
|
@@ -38,6 +38,8 @@ Requires-Dist: python-dotenv>=1.0.0
|
|
|
38
38
|
Requires-Dist: pyyaml>=6.0.0
|
|
39
39
|
Requires-Dist: prompt_toolkit>=3.0.0
|
|
40
40
|
Requires-Dist: textual>=0.40.0
|
|
41
|
+
Requires-Dist: pandas>=2.0.0
|
|
42
|
+
Requires-Dist: pyarrow>=12.0.0
|
|
41
43
|
Provides-Extra: dev
|
|
42
44
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
43
45
|
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
|
@@ -88,7 +90,10 @@ dremio view create --path "Analytics.summary" \
|
|
|
88
90
|
|
|
89
91
|
## 📚 Documentation
|
|
90
92
|
|
|
91
|
-
**[Complete Documentation →](docs
|
|
93
|
+
**[Complete Documentation (GitHub) →](https://github.com/developer-advocacy-dremio/dremio-python-cli/tree/main/dremio-cli/docs)**
|
|
94
|
+
|
|
95
|
+
*(Use the link above if browsing on PyPI)*
|
|
96
|
+
|
|
92
97
|
|
|
93
98
|
### Core Guides
|
|
94
99
|
|
|
@@ -118,6 +123,11 @@ dremio view create --path "Analytics.summary" \
|
|
|
118
123
|
|
|
119
124
|
- **[Favorites](docs/favorites.md)** - Save frequently used queries
|
|
120
125
|
- **[History](docs/history.md)** - View and re-run commands
|
|
126
|
+
- **[REPL](docs/repl.md)** - Interactive SQL Shell (Enhanced)
|
|
127
|
+
- **[Monitor](docs/monitor.md)** - Real-time System Monitor
|
|
128
|
+
- **[Init](docs/init.md)** - Interactive Configuration Wizard
|
|
129
|
+
- **[Completion](docs/completion.md)** - Shell Completion Scripts
|
|
130
|
+
- **[Lineage](docs/lineage.md)** - Visualize dataset dependencies
|
|
121
131
|
|
|
122
132
|
## 🎯 Key Features
|
|
123
133
|
|
|
@@ -37,7 +37,10 @@ dremio view create --path "Analytics.summary" \
|
|
|
37
37
|
|
|
38
38
|
## 📚 Documentation
|
|
39
39
|
|
|
40
|
-
**[Complete Documentation →](docs
|
|
40
|
+
**[Complete Documentation (GitHub) →](https://github.com/developer-advocacy-dremio/dremio-python-cli/tree/main/dremio-cli/docs)**
|
|
41
|
+
|
|
42
|
+
*(Use the link above if browsing on PyPI)*
|
|
43
|
+
|
|
41
44
|
|
|
42
45
|
### Core Guides
|
|
43
46
|
|
|
@@ -67,6 +70,11 @@ dremio view create --path "Analytics.summary" \
|
|
|
67
70
|
|
|
68
71
|
- **[Favorites](docs/favorites.md)** - Save frequently used queries
|
|
69
72
|
- **[History](docs/history.md)** - View and re-run commands
|
|
73
|
+
- **[REPL](docs/repl.md)** - Interactive SQL Shell (Enhanced)
|
|
74
|
+
- **[Monitor](docs/monitor.md)** - Real-time System Monitor
|
|
75
|
+
- **[Init](docs/init.md)** - Interactive Configuration Wizard
|
|
76
|
+
- **[Completion](docs/completion.md)** - Shell Completion Scripts
|
|
77
|
+
- **[Lineage](docs/lineage.md)** - Visualize dataset dependencies
|
|
70
78
|
|
|
71
79
|
## 🎯 Key Features
|
|
72
80
|
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Dremio-as-Code (GitOps)
|
|
2
|
+
|
|
3
|
+
Dremio-as-Code (DAC) allows you to manage your Dremio catalog (Spaces, Folders, Views) using local files, enabling GitOps workflows.
|
|
4
|
+
|
|
5
|
+
## Quick Start
|
|
6
|
+
|
|
7
|
+
### 1. Initialize Configuration
|
|
8
|
+
Create a `dremio.yaml` file in your project root:
|
|
9
|
+
|
|
10
|
+
```yaml
|
|
11
|
+
version: "1.0"
|
|
12
|
+
scope:
|
|
13
|
+
path: "dremio-catalog.finance" # The Dremio path to sync
|
|
14
|
+
type: "SPACE" # SPACE or ICEBERGCATALOG
|
|
15
|
+
ignore:
|
|
16
|
+
- "*.tmp"
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
### 2. Pull State
|
|
20
|
+
Capture the current state of your Dremio space into local files.
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
dremio sync pull
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Returns a directory structure mirroring Dremio:
|
|
27
|
+
- `finance/`
|
|
28
|
+
- `monthly_report.sql`
|
|
29
|
+
- `monthly_report.yaml`
|
|
30
|
+
|
|
31
|
+
### 3. Make Changes
|
|
32
|
+
Edit a view's SQL locally or create a new file pair (`.sql` and `.yaml`).
|
|
33
|
+
|
|
34
|
+
**new_view.sql**:
|
|
35
|
+
```sql
|
|
36
|
+
SELECT * FROM parent.table
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
**new_view.yaml**:
|
|
40
|
+
```yaml
|
|
41
|
+
type: VIEW
|
|
42
|
+
path: ["dremio-catalog", "finance", "new_view"]
|
|
43
|
+
sql_file: "new_view.sql"
|
|
44
|
+
context: []
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
### 4. Push Changes
|
|
48
|
+
Apply your local changes back to Dremio.
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
dremio sync push
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Concepts
|
|
55
|
+
|
|
56
|
+
- **Scope**: Limits the sync to a specific subtree to support multi-team environments.
|
|
57
|
+
- **State File**: `.dremio_state.json` tracks the last known state to enable efficient updates and deletes. **Do not edit this file manually.**
|
|
58
|
+
- **File Format**:
|
|
59
|
+
- **Context**: List of path parts to set query context.
|
|
60
|
+
- **Reflections**: (Coming Soon) Defined in YAML.
|
|
@@ -60,6 +60,19 @@ dremio --output json job get 16b2c9cd-a920-952b-b162-2280c9059d00
|
|
|
60
60
|
dremio --verbose job get 16b2c9cd-a920-952b-b162-2280c9059d00
|
|
61
61
|
```
|
|
62
62
|
|
|
63
|
+
### Analyze Job
|
|
64
|
+
|
|
65
|
+
Analyze the performance of a job to identify bottlenecks.
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
dremio job analyze <JOB_ID>
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
**Output:**
|
|
72
|
+
- Job duration and state
|
|
73
|
+
- Data reduction ratio (input vs output records)
|
|
74
|
+
- Insights on performance metrics
|
|
75
|
+
|
|
63
76
|
### Get Job Results
|
|
64
77
|
|
|
65
78
|
Retrieve the results of a completed job.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Lineage Visualization
|
|
2
|
+
|
|
3
|
+
Visualize the dependencies between datasets.
|
|
4
|
+
|
|
5
|
+
## Commands
|
|
6
|
+
|
|
7
|
+
### Show Lineage
|
|
8
|
+
|
|
9
|
+
Show the upstream parents of a dataset.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
dremio lineage show <CATALOG_ID> [OPTIONS]
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
**Options:**
|
|
16
|
+
- `--format [tree|json|mermaid]` - Output format (Default: tree)
|
|
17
|
+
|
|
18
|
+
**Examples:**
|
|
19
|
+
```bash
|
|
20
|
+
# Tree view (Terminal)
|
|
21
|
+
dremio lineage show dremio-catalog.space.view
|
|
22
|
+
|
|
23
|
+
# Mermaid Graph (for markdown)
|
|
24
|
+
dremio lineage show dremio-catalog.space.view --format mermaid
|
|
25
|
+
```
|
|
@@ -20,7 +20,9 @@ dremio sql execute --file <FILE> [OPTIONS]
|
|
|
20
20
|
- `--file PATH` - Execute SQL from file
|
|
21
21
|
- `--context TEXT` - Query context (comma-separated path)
|
|
22
22
|
- `--async` - Execute asynchronously (return job ID immediately). Default behavior waits for results.
|
|
23
|
-
- `--output-file PATH` - Save results to file
|
|
23
|
+
- `--output-file PATH` - Save results to file (supports .json, .yaml, .csv, .parquet)
|
|
24
|
+
|
|
25
|
+
**Note:** `.csv` and `.parquet` export require `pandas` and `pyarrow` installed.
|
|
24
26
|
|
|
25
27
|
**Examples:**
|
|
26
28
|
|
|
@@ -30,6 +30,8 @@ from dremio_cli.commands import (
|
|
|
30
30
|
repl,
|
|
31
31
|
monitor,
|
|
32
32
|
completion,
|
|
33
|
+
lineage,
|
|
34
|
+
sync,
|
|
33
35
|
)
|
|
34
36
|
|
|
35
37
|
|
|
@@ -102,6 +104,8 @@ cli.add_command(init.init_command)
|
|
|
102
104
|
cli.add_command(repl.repl_command)
|
|
103
105
|
cli.add_command(monitor.monitor_command)
|
|
104
106
|
cli.add_command(completion.completion_command)
|
|
107
|
+
cli.add_command(lineage.lineage)
|
|
108
|
+
cli.add_command(sync.sync)
|
|
105
109
|
|
|
106
110
|
|
|
107
111
|
|
|
@@ -363,3 +363,9 @@ class CloudClient(BaseClient):
|
|
|
363
363
|
"""Delete a script."""
|
|
364
364
|
return self.delete(self._project_endpoint(f"scripts/{script_id}"))
|
|
365
365
|
|
|
366
|
+
# Lineage operations
|
|
367
|
+
def get_catalog_graph(self, catalog_id: str) -> Dict[str, Any]:
|
|
368
|
+
"""Get lineage graph for a catalog item."""
|
|
369
|
+
# Cloud API usually follows similar pattern or project endpoint
|
|
370
|
+
return self.get(self._project_endpoint(f"catalog/{catalog_id}/graph"))
|
|
371
|
+
|
|
@@ -609,10 +609,10 @@ class SoftwareClient(BaseClient):
|
|
|
609
609
|
return self.put(f"reflection/{reflection_id}", data=reflection_data)
|
|
610
610
|
|
|
611
611
|
def delete_reflection(self, reflection_id: str) -> None:
|
|
612
|
-
"""Delete a reflection.
|
|
613
|
-
|
|
614
|
-
Args:
|
|
615
|
-
reflection_id: Reflection ID
|
|
616
|
-
"""
|
|
612
|
+
"""Delete a reflection."""
|
|
617
613
|
return self.delete(f"reflection/{reflection_id}")
|
|
618
614
|
|
|
615
|
+
# Lineage operations
|
|
616
|
+
def get_catalog_graph(self, catalog_id: str) -> Dict[str, Any]:
|
|
617
|
+
"""Get lineage graph for a catalog item."""
|
|
618
|
+
return self.get(f"catalog/{catalog_id}/graph")
|
|
@@ -319,3 +319,85 @@ def get_job_reflections(ctx, job_id: str) -> None:
|
|
|
319
319
|
import traceback
|
|
320
320
|
traceback.print_exc()
|
|
321
321
|
raise click.Abort()
|
|
322
|
+
@job.command("analyze")
|
|
323
|
+
@click.argument("job_id")
|
|
324
|
+
@click.pass_context
|
|
325
|
+
def analyze_job(ctx, job_id: str) -> None:
|
|
326
|
+
"""Analyze job performance.
|
|
327
|
+
|
|
328
|
+
Examples:
|
|
329
|
+
dremio job analyze <job-id>
|
|
330
|
+
"""
|
|
331
|
+
try:
|
|
332
|
+
manager = ProfileManager()
|
|
333
|
+
profile_name = ctx.obj.profile_name
|
|
334
|
+
profile = manager.get_profile(profile_name)
|
|
335
|
+
|
|
336
|
+
if not profile:
|
|
337
|
+
console.print(f"[red]Profile '{profile_name}' not found[/red]")
|
|
338
|
+
return
|
|
339
|
+
|
|
340
|
+
client = create_client(profile)
|
|
341
|
+
|
|
342
|
+
with console.status(f"[bold green]Analyzing job {job_id}..."):
|
|
343
|
+
job = client.get_job(job_id)
|
|
344
|
+
|
|
345
|
+
# Basic info
|
|
346
|
+
state = job.get("jobState")
|
|
347
|
+
start = job.get("startTime")
|
|
348
|
+
end = job.get("endTime")
|
|
349
|
+
|
|
350
|
+
# Stats (structure varies between Cloud/Software versions)
|
|
351
|
+
# Safe access to stats
|
|
352
|
+
stats = job.get("stats", {})
|
|
353
|
+
|
|
354
|
+
# Provide insights
|
|
355
|
+
from rich.panel import Panel
|
|
356
|
+
from rich.columns import Columns
|
|
357
|
+
|
|
358
|
+
# Duration
|
|
359
|
+
duration = "N/A"
|
|
360
|
+
if start and end:
|
|
361
|
+
# Basic duration calc if not in stats (assuming epoch ms)
|
|
362
|
+
try:
|
|
363
|
+
diff = (int(end) - int(start)) / 1000.0
|
|
364
|
+
duration = f"{diff:.2f}s"
|
|
365
|
+
except:
|
|
366
|
+
pass
|
|
367
|
+
|
|
368
|
+
# Input/Output
|
|
369
|
+
input_bytes = stats.get("inputBytes", 0)
|
|
370
|
+
output_bytes = stats.get("outputBytes", 0)
|
|
371
|
+
|
|
372
|
+
# Simple insights
|
|
373
|
+
insights = []
|
|
374
|
+
if state != "COMPLETED":
|
|
375
|
+
insights.append(f"[red]Job did not complete ({state})[/red]")
|
|
376
|
+
|
|
377
|
+
input_records = stats.get("inputRecords", 0)
|
|
378
|
+
output_records = stats.get("outputRecords", 0)
|
|
379
|
+
if input_records > 0 and output_records > 0:
|
|
380
|
+
reduction = (1 - (output_records / input_records)) * 100
|
|
381
|
+
insights.append(f"[green]Data Reduction: {reduction:.1f}%[/green]")
|
|
382
|
+
|
|
383
|
+
# Display
|
|
384
|
+
console.print(Panel(f"""
|
|
385
|
+
[bold]Job ID[/bold]: {job_id}
|
|
386
|
+
[bold]State[/bold]: {state}
|
|
387
|
+
[bold]Duration[/bold]: {duration}
|
|
388
|
+
[bold]User[/bold]: {job.get('user', 'N/A')}
|
|
389
|
+
""", title="Job Summary", border_style="cyan"))
|
|
390
|
+
|
|
391
|
+
if insights:
|
|
392
|
+
console.print(Panel("\n".join(insights), title="Insights", border_style="yellow"))
|
|
393
|
+
|
|
394
|
+
# Raw stats table
|
|
395
|
+
from rich.table import Table
|
|
396
|
+
t = Table(title="Metrics", show_header=False)
|
|
397
|
+
t.add_row("Input Records", str(input_records))
|
|
398
|
+
t.add_row("Output Records", str(output_records))
|
|
399
|
+
t.add_row("Memory Used", f"{stats.get('memoryAllocation', 0)} bytes")
|
|
400
|
+
console.print(t)
|
|
401
|
+
|
|
402
|
+
except Exception as e:
|
|
403
|
+
console.print(f"[red]Analysis failed: {e}[/red]")
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Lineage management commands."""
|
|
2
|
+
|
|
3
|
+
import click
|
|
4
|
+
import json
|
|
5
|
+
from rich.console import Console
|
|
6
|
+
from rich.tree import Tree
|
|
7
|
+
|
|
8
|
+
from dremio_cli.config import ProfileManager
|
|
9
|
+
from dremio_cli.client.factory import create_client
|
|
10
|
+
from dremio_cli.formatters.json import format_as_json
|
|
11
|
+
|
|
12
|
+
console = Console()
|
|
13
|
+
|
|
14
|
+
@click.group()
|
|
15
|
+
def lineage():
|
|
16
|
+
"""Lineage visualization operations."""
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
@lineage.command("show")
|
|
20
|
+
@click.argument("catalog_id")
|
|
21
|
+
@click.option("--format", type=click.Choice(["tree", "json", "mermaid"], case_sensitive=False), default="tree", help="Output format")
|
|
22
|
+
@click.pass_context
|
|
23
|
+
def show_lineage(ctx, catalog_id: str, format: str):
|
|
24
|
+
"""Show lineage graph for a dataset.
|
|
25
|
+
|
|
26
|
+
Examples:
|
|
27
|
+
dremio lineage show <catalog-id>
|
|
28
|
+
dremio lineage show <catalog-id> --format mermaid
|
|
29
|
+
"""
|
|
30
|
+
try:
|
|
31
|
+
manager = ProfileManager()
|
|
32
|
+
profile_name = ctx.obj.profile_name
|
|
33
|
+
profile = manager.get_profile(profile_name)
|
|
34
|
+
|
|
35
|
+
if not profile:
|
|
36
|
+
console.print(f"[red]Profile '{profile_name}' not found[/red]")
|
|
37
|
+
return
|
|
38
|
+
|
|
39
|
+
client = create_client(profile)
|
|
40
|
+
|
|
41
|
+
try:
|
|
42
|
+
graph = client.get_catalog_graph(catalog_id)
|
|
43
|
+
except Exception as e:
|
|
44
|
+
console.print(f"[red]Error fetching lineage: {e}[/red]")
|
|
45
|
+
# Fallback or specific error handling (e.g. 404)
|
|
46
|
+
return
|
|
47
|
+
|
|
48
|
+
if format == "json":
|
|
49
|
+
console.print(format_as_json(graph))
|
|
50
|
+
return
|
|
51
|
+
|
|
52
|
+
# Process graph for Tree/Mermaid
|
|
53
|
+
# Graph structure usually: { "parents": [...], "children": [...] }
|
|
54
|
+
# Or detailed nodes/edges depending on API version.
|
|
55
|
+
# Assuming Dremio V3 graph response structure:
|
|
56
|
+
# { "parents": [ { "id": "...", "path": [...] } ], "children": [] }
|
|
57
|
+
|
|
58
|
+
# We focus on parents (upstream lineage)
|
|
59
|
+
parents = graph.get("parents", [])
|
|
60
|
+
|
|
61
|
+
if format == "mermaid":
|
|
62
|
+
click.echo("graph TD")
|
|
63
|
+
# Current node
|
|
64
|
+
click.echo(f' root["{catalog_id}"]')
|
|
65
|
+
click.echo(" style root fill:#f9f,stroke:#333,stroke-width:2px")
|
|
66
|
+
|
|
67
|
+
for parent in parents:
|
|
68
|
+
p_path = ".".join(parent.get("path", []))
|
|
69
|
+
p_id = parent.get("id", p_path) # fallback if id missing
|
|
70
|
+
# Sanitize ID for mermaid
|
|
71
|
+
safe_pid = p_id.replace("-", "_").replace(".", "_")
|
|
72
|
+
|
|
73
|
+
click.echo(f' {safe_pid}["{p_path}"] --> root')
|
|
74
|
+
|
|
75
|
+
else: # tree
|
|
76
|
+
tree = Tree(f"[bold cyan]{catalog_id}[/bold cyan]")
|
|
77
|
+
if parents:
|
|
78
|
+
p_branch = tree.add("Parents")
|
|
79
|
+
for parent in parents:
|
|
80
|
+
p_path = ".".join(parent.get("path", []))
|
|
81
|
+
p_branch.add(f"[green]{p_path}[/green]")
|
|
82
|
+
else:
|
|
83
|
+
tree.add("[italic]No parents found[/italic]")
|
|
84
|
+
|
|
85
|
+
console.print(tree)
|
|
86
|
+
|
|
87
|
+
except Exception as e:
|
|
88
|
+
console.print(f"[red]Error: {e}[/red]")
|
|
@@ -148,7 +148,26 @@ def execute_sql(ctx, query: str, sql_file: str, context: str, async_mode: bool,
|
|
|
148
148
|
console.print("[yellow]Warning: --output-file with multiple queries will subscribe/overwite. Currently not fully supported for multi-statement.[/yellow]")
|
|
149
149
|
|
|
150
150
|
output_path = Path(output_file)
|
|
151
|
-
|
|
151
|
+
|
|
152
|
+
# Advanced Formats (Pandas required)
|
|
153
|
+
if output_path.suffix in ['.parquet', '.csv']:
|
|
154
|
+
try:
|
|
155
|
+
import pandas as pd
|
|
156
|
+
df = pd.DataFrame(rows)
|
|
157
|
+
|
|
158
|
+
if output_path.suffix == '.parquet':
|
|
159
|
+
df.to_parquet(output_path)
|
|
160
|
+
else: # csv
|
|
161
|
+
df.to_csv(output_path, index=False)
|
|
162
|
+
|
|
163
|
+
except ImportError:
|
|
164
|
+
console.print("[red]Error: 'pandas' and 'pyarrow' are required for Parquet/CSV export. Install them with pip.[/red]")
|
|
165
|
+
raise click.Abort()
|
|
166
|
+
except Exception as e:
|
|
167
|
+
console.print(f"[red]Export failed: {e}[/red]")
|
|
168
|
+
raise click.Abort()
|
|
169
|
+
|
|
170
|
+
elif output_path.suffix == '.json':
|
|
152
171
|
output_path.write_text(json.dumps(results, indent=2))
|
|
153
172
|
elif output_path.suffix in ['.yaml', '.yml']:
|
|
154
173
|
import yaml
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
|
|
2
|
+
import os
|
|
3
|
+
import click
|
|
4
|
+
from dremio_cli.utils.console import console
|
|
5
|
+
from dremio_cli.dac.config import DremioConfig
|
|
6
|
+
from dremio_cli.dac.sync import DremioSync
|
|
7
|
+
|
|
8
|
+
@click.group(help="Dremio-as-Code synchronization.")
|
|
9
|
+
def sync():
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
@sync.command("pull")
|
|
13
|
+
@click.option("--config", default="dremio.yaml", help="Path to dremio.yaml config file.")
|
|
14
|
+
@click.pass_context
|
|
15
|
+
def pull(ctx, config):
|
|
16
|
+
"""Pull state from Dremio to local filesystem."""
|
|
17
|
+
try:
|
|
18
|
+
if not os.path.exists(config):
|
|
19
|
+
console.print(f"[red]Config file '{config}' not found. Please create a dremio.yaml first.[/red]")
|
|
20
|
+
return
|
|
21
|
+
|
|
22
|
+
conf = DremioConfig.load(config)
|
|
23
|
+
client = ctx.obj
|
|
24
|
+
|
|
25
|
+
syncer = DremioSync(conf, client, os.path.dirname(os.path.abspath(config)))
|
|
26
|
+
syncer.pull()
|
|
27
|
+
|
|
28
|
+
except Exception as e:
|
|
29
|
+
console.print(f"[red]Pull failed: {e}[/red]")
|
|
30
|
+
# import traceback
|
|
31
|
+
# traceback.print_exc()
|
|
32
|
+
|
|
33
|
+
@sync.command("push")
|
|
34
|
+
@click.option("--config", default="dremio.yaml", help="Path to dremio.yaml config file.")
|
|
35
|
+
@click.option("--dry-run", is_flag=True, help="Show changes without applying them.")
|
|
36
|
+
@click.pass_context
|
|
37
|
+
def push(ctx, config, dry_run):
|
|
38
|
+
"""Push local state to Dremio."""
|
|
39
|
+
try:
|
|
40
|
+
if not os.path.exists(config):
|
|
41
|
+
console.print(f"[red]Config file '{config}' not found.[/red]")
|
|
42
|
+
return
|
|
43
|
+
|
|
44
|
+
conf = DremioConfig.load(config)
|
|
45
|
+
client = ctx.obj
|
|
46
|
+
|
|
47
|
+
syncer = DremioSync(conf, client, os.path.dirname(os.path.abspath(config)))
|
|
48
|
+
syncer.push(dry_run=dry_run)
|
|
49
|
+
|
|
50
|
+
except Exception as e:
|
|
51
|
+
console.print(f"[red]Push failed: {e}[/red]")
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
|
|
2
|
+
import os
|
|
3
|
+
import yaml
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class Scope:
|
|
9
|
+
path: str
|
|
10
|
+
type: str # SPACE, ICEBERGCATALOG, SOURCE
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class DremioConfig:
|
|
14
|
+
version: str
|
|
15
|
+
scope: Scope
|
|
16
|
+
ignore: List[str]
|
|
17
|
+
|
|
18
|
+
@classmethod
|
|
19
|
+
def load(cls, path: str = "dremio.yaml") -> 'DremioConfig':
|
|
20
|
+
if not os.path.exists(path):
|
|
21
|
+
raise FileNotFoundError(f"Config file {path} not found.")
|
|
22
|
+
|
|
23
|
+
with open(path, "r") as f:
|
|
24
|
+
data = yaml.safe_load(f)
|
|
25
|
+
|
|
26
|
+
scope_data = data.get("scope", {})
|
|
27
|
+
if not scope_data.get("path") or not scope_data.get("type"):
|
|
28
|
+
raise ValueError("dremio.yaml must define scope.path and scope.type")
|
|
29
|
+
|
|
30
|
+
# Validate type
|
|
31
|
+
valid_types = ["SPACE", "ICEBERGCATALOG", "SOURCE"]
|
|
32
|
+
if scope_data["type"] not in valid_types:
|
|
33
|
+
raise ValueError(f"Invalid scope.type: {scope_data['type']}. Must be one of {valid_types}")
|
|
34
|
+
|
|
35
|
+
return cls(
|
|
36
|
+
version=data.get("version", "1.0"),
|
|
37
|
+
scope=Scope(
|
|
38
|
+
path=scope_data["path"],
|
|
39
|
+
type=scope_data["type"]
|
|
40
|
+
),
|
|
41
|
+
ignore=data.get("ignore", [])
|
|
42
|
+
)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
|
|
2
|
+
from typing import List, Dict, Any
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
@dataclass
|
|
6
|
+
class DiffItem:
|
|
7
|
+
action: str # CREATE, UPDATE, DELETE, NOOP
|
|
8
|
+
path: List[str]
|
|
9
|
+
type: str
|
|
10
|
+
reason: str
|
|
11
|
+
local_data: Dict[str, Any] = None
|
|
12
|
+
|
|
13
|
+
class Differ:
|
|
14
|
+
def __init__(self, local_state: 'LocalState', remote_state: Dict[str, Any]):
|
|
15
|
+
"""
|
|
16
|
+
local_state: The LocalState object managing the .dremio_state.json
|
|
17
|
+
remote_state: A dictionary representing the current structure on Dremio side (fetched via API)
|
|
18
|
+
"""
|
|
19
|
+
self.local = local_state
|
|
20
|
+
self.remote = remote_state
|
|
21
|
+
|
|
22
|
+
# TODO: Implement diff logic in the Push implementation phase
|
|
23
|
+
# This serves as a placeholder for the module structure.
|
|
24
|
+
def compare(self) -> List[DiffItem]:
|
|
25
|
+
diffs = []
|
|
26
|
+
return diffs
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
|
|
2
|
+
import os
|
|
3
|
+
import json
|
|
4
|
+
import yaml
|
|
5
|
+
from typing import Dict, Any, List, Optional
|
|
6
|
+
from dataclasses import dataclass, asdict
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class ResourceState:
|
|
10
|
+
id: str
|
|
11
|
+
path: List[str]
|
|
12
|
+
type: str # DATASET, FOLDER, SPACE, SOURCE, VIEW (Specific)
|
|
13
|
+
hash: str # Content hash for change detection
|
|
14
|
+
metadata: Dict[str, Any]
|
|
15
|
+
|
|
16
|
+
class LocalState:
|
|
17
|
+
def __init__(self, root_path: str):
|
|
18
|
+
self.root_path = root_path
|
|
19
|
+
self.state_file = os.path.join(root_path, ".dremio_state.json")
|
|
20
|
+
self.resources: Dict[str, ResourceState] = {} # Keyed by path string "space.folder.view"
|
|
21
|
+
|
|
22
|
+
def load(self):
|
|
23
|
+
"""Load state from .dremio_state.json"""
|
|
24
|
+
if os.path.exists(self.state_file):
|
|
25
|
+
try:
|
|
26
|
+
with open(self.state_file, 'r') as f:
|
|
27
|
+
data = json.load(f)
|
|
28
|
+
for key, item in data.items():
|
|
29
|
+
self.resources[key] = ResourceState(**item)
|
|
30
|
+
except Exception as e:
|
|
31
|
+
print(f"Warning: Could not load state file: {e}")
|
|
32
|
+
|
|
33
|
+
def save(self):
|
|
34
|
+
"""Save state to .dremio_state.json"""
|
|
35
|
+
data = {k: asdict(v) for k, v in self.resources.items()}
|
|
36
|
+
with open(self.state_file, 'w') as f:
|
|
37
|
+
json.dump(data, f, indent=2)
|
|
38
|
+
|
|
39
|
+
def update_resource(self, path_list: List[str], type: str, id: str, hash: str, metadata: Dict[str, Any]):
|
|
40
|
+
key = ".".join(path_list)
|
|
41
|
+
self.resources[key] = ResourceState(
|
|
42
|
+
id=id,
|
|
43
|
+
path=path_list,
|
|
44
|
+
type=type,
|
|
45
|
+
hash=hash,
|
|
46
|
+
metadata=metadata
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
def get_resource(self, path_list: List[str]) -> Optional[ResourceState]:
|
|
50
|
+
key = ".".join(path_list)
|
|
51
|
+
return self.resources.get(key)
|
|
52
|
+
|
|
53
|
+
def remove_resource(self, path_list: List[str]):
|
|
54
|
+
key = ".".join(path_list)
|
|
55
|
+
if key in self.resources:
|
|
56
|
+
del self.resources[key]
|
|
57
|
+
|
|
58
|
+
def scan_filesystem(self) -> Dict[str, Dict[str, Any]]:
|
|
59
|
+
"""
|
|
60
|
+
Scans the root_dir for .yaml and .sql files to build a picture of local state.
|
|
61
|
+
Returns a dict of resource definitions found on disk.
|
|
62
|
+
"""
|
|
63
|
+
local_resources = {}
|
|
64
|
+
for root, dirs, files in os.walk(self.root_path):
|
|
65
|
+
# skipping .git, .dremio_state.json, etc is handled by not parsing them
|
|
66
|
+
for file in files:
|
|
67
|
+
if file.endswith(".yaml") and file != "dremio.yaml":
|
|
68
|
+
full_path = os.path.join(root, file)
|
|
69
|
+
try:
|
|
70
|
+
with open(full_path, 'r') as f:
|
|
71
|
+
data = yaml.safe_load(f)
|
|
72
|
+
# We expect 'type' and 'path' in the yaml
|
|
73
|
+
if data and "type" in data and "path" in data:
|
|
74
|
+
path_key = ".".join(data["path"])
|
|
75
|
+
local_resources[path_key] = data
|
|
76
|
+
# If there is a sql file, read it
|
|
77
|
+
if "sql_file" in data:
|
|
78
|
+
sql_path = os.path.join(root, data["sql_file"])
|
|
79
|
+
if os.path.exists(sql_path):
|
|
80
|
+
with open(sql_path, 'r') as sql_f:
|
|
81
|
+
data["sql_content"] = sql_f.read()
|
|
82
|
+
except Exception as e:
|
|
83
|
+
print(f"Error reading {full_path}: {e}")
|
|
84
|
+
return local_resources
|