dremio-cli 1.5.0__tar.gz → 1.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. {dremio_cli-1.5.0/dremio_cli.egg-info → dremio_cli-1.7.0}/PKG-INFO +13 -2
  2. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/README.md +10 -1
  3. dremio_cli-1.7.0/docs/dac.md +60 -0
  4. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/jobs.md +13 -0
  5. dremio_cli-1.7.0/docs/lineage.md +25 -0
  6. dremio_cli-1.7.0/docs/profiles.md +139 -0
  7. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/sql.md +3 -1
  8. dremio_cli-1.7.0/docs/tui.md +44 -0
  9. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/cli.py +6 -0
  10. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/auth.py +6 -1
  11. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/cloud.py +6 -0
  12. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/software.py +10 -5
  13. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/job.py +82 -0
  14. dremio_cli-1.7.0/dremio_cli/commands/lineage.py +88 -0
  15. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/sql.py +20 -1
  16. dremio_cli-1.7.0/dremio_cli/commands/sync.py +51 -0
  17. dremio_cli-1.7.0/dremio_cli/commands/ui.py +20 -0
  18. dremio_cli-1.7.0/dremio_cli/dac/__init__.py +3 -0
  19. dremio_cli-1.7.0/dremio_cli/dac/config.py +42 -0
  20. dremio_cli-1.7.0/dremio_cli/dac/diff.py +26 -0
  21. dremio_cli-1.7.0/dremio_cli/dac/state.py +84 -0
  22. dremio_cli-1.7.0/dremio_cli/dac/sync.py +344 -0
  23. dremio_cli-1.7.0/dremio_cli/tui/catalog/__init__.py +1 -0
  24. dremio_cli-1.7.0/dremio_cli/tui/catalog/app.py +34 -0
  25. dremio_cli-1.7.0/dremio_cli/tui/catalog/screens.py +117 -0
  26. dremio_cli-1.7.0/dremio_cli/tui/catalog/widgets.py +180 -0
  27. dremio_cli-1.7.0/dremio_cli/utils/console.py +4 -0
  28. {dremio_cli-1.5.0 → dremio_cli-1.7.0/dremio_cli.egg-info}/PKG-INFO +13 -2
  29. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli.egg-info/SOURCES.txt +15 -0
  30. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli.egg-info/requires.txt +2 -0
  31. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/pyproject.toml +3 -1
  32. dremio_cli-1.5.0/docs/profiles.md +0 -378
  33. dremio_cli-1.5.0/dremio_cli/commands/lineage.py +0 -9
  34. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/LICENSE +0 -0
  35. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/MANIFEST.in +0 -0
  36. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/completions/dremio-completion.bash +0 -0
  37. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/completions/dremio-completion.zsh +0 -0
  38. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/README.md +0 -0
  39. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/advanced-features.md +0 -0
  40. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/catalog.md +0 -0
  41. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/completion.md +0 -0
  42. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/favorites.md +0 -0
  43. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/grants.md +0 -0
  44. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/history.md +0 -0
  45. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/init.md +0 -0
  46. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/installation.md +0 -0
  47. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/monitor.md +0 -0
  48. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/quickstart.md +0 -0
  49. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/reflections.md +0 -0
  50. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/repl.md +0 -0
  51. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/roles.md +0 -0
  52. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/scripts.md +0 -0
  53. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/sources.md +0 -0
  54. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/spaces-folders.md +0 -0
  55. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/tables.md +0 -0
  56. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/tags-wiki.md +0 -0
  57. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/users.md +0 -0
  58. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/docs/views.md +0 -0
  59. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/__init__.py +0 -0
  60. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/__main__.py +0 -0
  61. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/__init__.py +0 -0
  62. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/base.py +0 -0
  63. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/client/factory.py +0 -0
  64. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/__init__.py +0 -0
  65. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/catalog.py +0 -0
  66. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/completion.py +0 -0
  67. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/favorite.py +0 -0
  68. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/folder.py +0 -0
  69. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/grant.py +0 -0
  70. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/history.py +0 -0
  71. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/init.py +0 -0
  72. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/monitor.py +0 -0
  73. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/profile.py +0 -0
  74. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/reflection.py +0 -0
  75. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/repl.py +0 -0
  76. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/role.py +0 -0
  77. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/script.py +0 -0
  78. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/source.py +0 -0
  79. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/space.py +0 -0
  80. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/table.py +0 -0
  81. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/tag.py +0 -0
  82. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/udf.py +0 -0
  83. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/user.py +0 -0
  84. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/view.py +0 -0
  85. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/commands/wiki.py +0 -0
  86. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/config.py +0 -0
  87. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/env.py +0 -0
  88. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/formatters/__init__.py +0 -0
  89. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/formatters/json.py +0 -0
  90. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/formatters/table.py +0 -0
  91. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/formatters/yaml.py +0 -0
  92. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/history.py +0 -0
  93. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/utils/__init__.py +0 -0
  94. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/utils/exceptions.py +0 -0
  95. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/utils/helpers.py +0 -0
  96. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli/utils/validators.py +0 -0
  97. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli.egg-info/dependency_links.txt +0 -0
  98. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli.egg-info/entry_points.txt +0 -0
  99. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/dremio_cli.egg-info/top_level.txt +0 -0
  100. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/setup.cfg +0 -0
  101. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/setup.py +0 -0
  102. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/__init__.py +0 -0
  103. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/conftest.py +0 -0
  104. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_config.py +0 -0
  105. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_grant_commands.py +0 -0
  106. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_history_favorite.py +0 -0
  107. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_integration.py +0 -0
  108. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_job_commands.py +0 -0
  109. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_multi_sql_execution.py +0 -0
  110. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_reflection_commands.py +0 -0
  111. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_role_commands.py +0 -0
  112. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_script_commands.py +0 -0
  113. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_source_commands.py +0 -0
  114. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_space_folder_commands.py +0 -0
  115. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_sql_commands.py +0 -0
  116. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_table_commands.py +0 -0
  117. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_tag_wiki_commands.py +0 -0
  118. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_user_commands.py +0 -0
  119. {dremio_cli-1.5.0 → dremio_cli-1.7.0}/tests/test_view_commands.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dremio-cli
3
- Version: 1.5.0
3
+ Version: 1.7.0
4
4
  Summary: A comprehensive command-line interface for Dremio Cloud and Dremio Software
5
5
  Home-page: https://github.com/your-org/dremio-cli
6
6
  Author: Your Name
@@ -38,6 +38,8 @@ Requires-Dist: python-dotenv>=1.0.0
38
38
  Requires-Dist: pyyaml>=6.0.0
39
39
  Requires-Dist: prompt_toolkit>=3.0.0
40
40
  Requires-Dist: textual>=0.40.0
41
+ Requires-Dist: pandas>=2.0.0
42
+ Requires-Dist: pyarrow>=12.0.0
41
43
  Provides-Extra: dev
42
44
  Requires-Dist: pytest>=7.0.0; extra == "dev"
43
45
  Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
@@ -88,7 +90,10 @@ dremio view create --path "Analytics.summary" \
88
90
 
89
91
  ## 📚 Documentation
90
92
 
91
- **[Complete Documentation →](docs/README.md)**
93
+ **[Complete Documentation (GitHub) →](https://github.com/developer-advocacy-dremio/dremio-python-cli/tree/main/dremio-cli/docs)**
94
+
95
+ *(Use the link above if browsing on PyPI)*
96
+
92
97
 
93
98
  ### Core Guides
94
99
 
@@ -118,6 +123,12 @@ dremio view create --path "Analytics.summary" \
118
123
 
119
124
  - **[Favorites](docs/favorites.md)** - Save frequently used queries
120
125
  - **[History](docs/history.md)** - View and re-run commands
126
+ - **[REPL](docs/repl.md)** - Interactive SQL Shell (Enhanced)
127
+ - **[Monitor](docs/monitor.md)** - Real-time System Monitor
128
+ - **[Catalog Explorer](docs/tui.md)** - Interactive Catalog TUI
129
+ - **[Init](docs/init.md)** - Interactive Configuration Wizard
130
+ - **[Completion](docs/completion.md)** - Shell Completion Scripts
131
+ - **[Lineage](docs/lineage.md)** - Visualize dataset dependencies
121
132
 
122
133
  ## 🎯 Key Features
123
134
 
@@ -37,7 +37,10 @@ dremio view create --path "Analytics.summary" \
37
37
 
38
38
  ## 📚 Documentation
39
39
 
40
- **[Complete Documentation →](docs/README.md)**
40
+ **[Complete Documentation (GitHub) →](https://github.com/developer-advocacy-dremio/dremio-python-cli/tree/main/dremio-cli/docs)**
41
+
42
+ *(Use the link above if browsing on PyPI)*
43
+
41
44
 
42
45
  ### Core Guides
43
46
 
@@ -67,6 +70,12 @@ dremio view create --path "Analytics.summary" \
67
70
 
68
71
  - **[Favorites](docs/favorites.md)** - Save frequently used queries
69
72
  - **[History](docs/history.md)** - View and re-run commands
73
+ - **[REPL](docs/repl.md)** - Interactive SQL Shell (Enhanced)
74
+ - **[Monitor](docs/monitor.md)** - Real-time System Monitor
75
+ - **[Catalog Explorer](docs/tui.md)** - Interactive Catalog TUI
76
+ - **[Init](docs/init.md)** - Interactive Configuration Wizard
77
+ - **[Completion](docs/completion.md)** - Shell Completion Scripts
78
+ - **[Lineage](docs/lineage.md)** - Visualize dataset dependencies
70
79
 
71
80
  ## 🎯 Key Features
72
81
 
@@ -0,0 +1,60 @@
1
+ # Dremio-as-Code (GitOps)
2
+
3
+ Dremio-as-Code (DAC) allows you to manage your Dremio catalog (Spaces, Folders, Views) using local files, enabling GitOps workflows.
4
+
5
+ ## Quick Start
6
+
7
+ ### 1. Initialize Configuration
8
+ Create a `dremio.yaml` file in your project root:
9
+
10
+ ```yaml
11
+ version: "1.0"
12
+ scope:
13
+ path: "dremio-catalog.finance" # The Dremio path to sync
14
+ type: "SPACE" # SPACE or ICEBERGCATALOG
15
+ ignore:
16
+ - "*.tmp"
17
+ ```
18
+
19
+ ### 2. Pull State
20
+ Capture the current state of your Dremio space into local files.
21
+
22
+ ```bash
23
+ dremio sync pull
24
+ ```
25
+
26
+ Returns a directory structure mirroring Dremio:
27
+ - `finance/`
28
+ - `monthly_report.sql`
29
+ - `monthly_report.yaml`
30
+
31
+ ### 3. Make Changes
32
+ Edit a view's SQL locally or create a new file pair (`.sql` and `.yaml`).
33
+
34
+ **new_view.sql**:
35
+ ```sql
36
+ SELECT * FROM parent.table
37
+ ```
38
+
39
+ **new_view.yaml**:
40
+ ```yaml
41
+ type: VIEW
42
+ path: ["dremio-catalog", "finance", "new_view"]
43
+ sql_file: "new_view.sql"
44
+ context: []
45
+ ```
46
+
47
+ ### 4. Push Changes
48
+ Apply your local changes back to Dremio.
49
+
50
+ ```bash
51
+ dremio sync push
52
+ ```
53
+
54
+ ## Concepts
55
+
56
+ - **Scope**: Limits the sync to a specific subtree to support multi-team environments.
57
+ - **State File**: `.dremio_state.json` tracks the last known state to enable efficient updates and deletes. **Do not edit this file manually.**
58
+ - **File Format**:
59
+ - **Context**: List of path parts to set query context.
60
+ - **Reflections**: (Coming Soon) Defined in YAML.
@@ -60,6 +60,19 @@ dremio --output json job get 16b2c9cd-a920-952b-b162-2280c9059d00
60
60
  dremio --verbose job get 16b2c9cd-a920-952b-b162-2280c9059d00
61
61
  ```
62
62
 
63
+ ### Analyze Job
64
+
65
+ Analyze the performance of a job to identify bottlenecks.
66
+
67
+ ```bash
68
+ dremio job analyze <JOB_ID>
69
+ ```
70
+
71
+ **Output:**
72
+ - Job duration and state
73
+ - Data reduction ratio (input vs output records)
74
+ - Insights on performance metrics
75
+
63
76
  ### Get Job Results
64
77
 
65
78
  Retrieve the results of a completed job.
@@ -0,0 +1,25 @@
1
+ # Lineage Visualization
2
+
3
+ Visualize the dependencies between datasets.
4
+
5
+ ## Commands
6
+
7
+ ### Show Lineage
8
+
9
+ Show the upstream parents of a dataset.
10
+
11
+ ```bash
12
+ dremio lineage show <CATALOG_ID> [OPTIONS]
13
+ ```
14
+
15
+ **Options:**
16
+ - `--format [tree|json|mermaid]` - Output format (Default: tree)
17
+
18
+ **Examples:**
19
+ ```bash
20
+ # Tree view (Terminal)
21
+ dremio lineage show dremio-catalog.space.view
22
+
23
+ # Mermaid Graph (for markdown)
24
+ dremio lineage show dremio-catalog.space.view --format mermaid
25
+ ```
@@ -0,0 +1,139 @@
1
+ # Profile Management Guide
2
+
3
+ This guide covers how to create and manage Dremio CLI profiles using both YAML configuration files and environment variables.
4
+
5
+ ## Overview
6
+
7
+ Profiles store connection information for Dremio instances. The CLI supports two methods:
8
+
9
+ 1. **YAML Configuration** - Stored in `~/.dremio/profiles.yaml` (RECOMMENDED for local use)
10
+ 2. **Environment Variables** - Loaded from `.env` file or shell environment (RECOMMENDED for CI/CD)
11
+
12
+ Environment variables take precedence over YAML profiles.
13
+
14
+ ---
15
+
16
+ ## 🚀 Profile Values Guide
17
+
18
+ ### 1. Base URL
19
+ The URL to your Dremio instance's API.
20
+
21
+ | Platform | Format | Example | Note |
22
+ |----------|--------|---------|------|
23
+ | **Dremio Cloud** | `https://api.dremio.cloud/v0` | `https://api.dremio.cloud/v0` | Used for US control plane |
24
+ | **Dremio Cloud (EU)** | `https://api.dremio.eu/v0` | `https://api.dremio.eu/v0` | Used for EU control plane |
25
+ | **Dremio Software** | `http(s)://<host>:<port>` | `https://dremio.company.com` | **Smart URL**: The CLI automatically appends `/api/v3` if you omit it. |
26
+ | **Local Software** | `http://localhost:9047` | `http://localhost:9047` | Defaults for local Docker/install |
27
+
28
+ > **Note:** For Dremio Software, you can provide `https://dremio.company.com` OR `https://dremio.company.com/api/v3`. The CLI handles both correctly.
29
+
30
+ ### 2. Authentication
31
+ How you log in to Dremio.
32
+
33
+ | Type | Platform | Description |
34
+ |------|----------|-------------|
35
+ | **PAT** (Token) | Cloud & Software | **Recommended**. Personal Access Token generated in User Settings. |
36
+ | **Services Account** | Cloud Only | Treats Client/Secret as a PAT for automation. |
37
+ | **Username/Password** | Software Only | Traditional login. **Less secure** than PAT. |
38
+
39
+ ### 3. Project ID (Cloud Only)
40
+ Can be found in the URL of your Dremio Cloud project.
41
+ - URL: `https://app.dremio.cloud/projectId/12345-abcde.../home`
42
+ - Project ID: `12345-abcde...`
43
+
44
+ ---
45
+
46
+ ## YAML Configuration
47
+
48
+ **Location**: `~/.dremio/profiles.yaml`
49
+
50
+ ### Examples
51
+
52
+ **Dremio Cloud (US)**
53
+ ```yaml
54
+ profiles:
55
+ cloud-prod:
56
+ type: cloud
57
+ base_url: https://api.dremio.cloud/v0
58
+ project_id: 788baab4-3c3b-42da-9f1d-5cc6dc03147d
59
+ auth:
60
+ type: pat
61
+ token: your-personal-access-token
62
+ ```
63
+
64
+ **Dremio Software (Corporate)**
65
+ ```yaml
66
+ profiles:
67
+ corp-dremio:
68
+ type: software
69
+ base_url: https://dremio.corp.com # CLI will add /api/v3 automatically
70
+ auth:
71
+ type: pat
72
+ token: your-personal-access-token
73
+ ```
74
+
75
+ **Dremio Software (Local/Docker)**
76
+ ```yaml
77
+ profiles:
78
+ local:
79
+ type: software
80
+ base_url: http://localhost:9047 # Default port
81
+ auth:
82
+ type: username_password
83
+ username: admin
84
+ password: password123
85
+ ```
86
+
87
+ ### CLI Commands
88
+
89
+ ```bash
90
+ # Interactve Wizard (Best for beginners)
91
+ dremio init
92
+
93
+ # Create manually
94
+ dremio profile create --name prod --type cloud ...
95
+ ```
96
+
97
+ ---
98
+
99
+ ## Environment Variable Configuration
100
+
101
+ Ideal for CI/CD pipelines or Docker containers.
102
+
103
+ ### Pattern
104
+ `DREMIO_{PROFILE_NAME}_KEY=VALUE`
105
+
106
+ ### Example `.env` File
107
+ ```bash
108
+ # Cloud Profile (Name: 'CLOUD')
109
+ DREMIO_CLOUD_TYPE=cloud
110
+ DREMIO_CLOUD_BASE_URL=https://api.dremio.cloud/v0
111
+ DREMIO_CLOUD_PROJECTID=788baab4-3c3b-42da-9f1d-5cc6dc03147d
112
+ DREMIO_CLOUD_TOKEN=s3JcLOqFTR...
113
+
114
+ # Software Profile (Name: 'PROD')
115
+ DREMIO_PROD_TYPE=software
116
+ DREMIO_PROD_BASE_URL=https://dremio.corp.com
117
+ DREMIO_PROD_TOKEN=Q/ToosxORA...
118
+ ```
119
+
120
+ ### Usage
121
+ ```bash
122
+ # Authenticates using DREMIO_PROD_* variables
123
+ dremio --profile prod catalog list
124
+ ```
125
+
126
+ ---
127
+
128
+ ## Profile Management
129
+
130
+ ```bash
131
+ # List all profiles
132
+ dremio profile list
133
+
134
+ # Show active profile details
135
+ dremio profile current
136
+
137
+ # Set default profile (so you don't need --profile flag)
138
+ dremio profile set-default cloud-prod
139
+ ```
@@ -20,7 +20,9 @@ dremio sql execute --file <FILE> [OPTIONS]
20
20
  - `--file PATH` - Execute SQL from file
21
21
  - `--context TEXT` - Query context (comma-separated path)
22
22
  - `--async` - Execute asynchronously (return job ID immediately). Default behavior waits for results.
23
- - `--output-file PATH` - Save results to file
23
+ - `--output-file PATH` - Save results to file (supports .json, .yaml, .csv, .parquet)
24
+
25
+ **Note:** `.csv` and `.parquet` export require `pandas` and `pyarrow` installed.
24
26
 
25
27
  **Examples:**
26
28
 
@@ -0,0 +1,44 @@
1
+ # Catalog Explorer (TUI)
2
+
3
+ The Dremio CLI includes an interactive Terminal User Interface (TUI) for exploring your Dremio catalog, viewing schemas, and previewing data without leaving the command line.
4
+
5
+ ## Usage
6
+
7
+ ```bash
8
+ dremio ui catalog
9
+ ```
10
+
11
+ Or using a specific profile:
12
+
13
+ ```bash
14
+ dremio --profile prod ui catalog
15
+ ```
16
+
17
+ ## Features
18
+
19
+ ### 1. Interactive Navigation
20
+ - **Tree View**: Navigate seamlessly through Spaces, Sources, Folders, and your Home directory.
21
+ - **Async Loading**: Large catalogs load quickly as you expand nodes.
22
+
23
+ ### 2. Dataset Details
24
+ Select any dataset (View/Table) to see:
25
+ - **Info**: Metadata, ID, and Type.
26
+ - **Schema**: Column names and types.
27
+ - **SQL**: The underlying SQL definition (for Views).
28
+
29
+ ### 3. Data Preview
30
+ - Switch to the **Preview** tab to instantly run a query (`SELECT * ... LIMIT 20`) and view actual data samples in a table.
31
+
32
+ ## Controls
33
+
34
+ | Key | Action |
35
+ |-----|--------|
36
+ | `↑` / `↓` | Navigate Tree |
37
+ | `Space` | Expand/Collapse Folder |
38
+ | `Enter` | Select Item |
39
+ | `Click` | Select tabs (Info, Schema, Preview) |
40
+ | `q` | Quit |
41
+
42
+ ## Requirements
43
+ - Dremio Software or Dremio Cloud
44
+ - Terminal with UTF-8 support
@@ -30,6 +30,9 @@ from dremio_cli.commands import (
30
30
  repl,
31
31
  monitor,
32
32
  completion,
33
+ lineage,
34
+ sync,
35
+ ui,
33
36
  )
34
37
 
35
38
 
@@ -102,6 +105,9 @@ cli.add_command(init.init_command)
102
105
  cli.add_command(repl.repl_command)
103
106
  cli.add_command(monitor.monitor_command)
104
107
  cli.add_command(completion.completion_command)
108
+ cli.add_command(lineage.lineage)
109
+ cli.add_command(sync.sync)
110
+ cli.add_command(ui.ui)
105
111
 
106
112
 
107
113
 
@@ -25,7 +25,12 @@ def authenticate_with_username_password(
25
25
  AuthenticationError: If authentication fails
26
26
  """
27
27
  # Use v2 API for login
28
- login_url = base_url.replace("/api/v3", "/apiv2/login")
28
+ # Normalize base_url to get root
29
+ root_url = base_url.rstrip("/")
30
+ if root_url.endswith("/api/v3"):
31
+ root_url = root_url[:-7] # remove /api/v3
32
+
33
+ login_url = f"{root_url}/apiv2/login"
29
34
 
30
35
  try:
31
36
  response = requests.post(
@@ -363,3 +363,9 @@ class CloudClient(BaseClient):
363
363
  """Delete a script."""
364
364
  return self.delete(self._project_endpoint(f"scripts/{script_id}"))
365
365
 
366
+ # Lineage operations
367
+ def get_catalog_graph(self, catalog_id: str) -> Dict[str, Any]:
368
+ """Get lineage graph for a catalog item."""
369
+ # Cloud API usually follows similar pattern or project endpoint
370
+ return self.get(self._project_endpoint(f"catalog/{catalog_id}/graph"))
371
+
@@ -21,6 +21,11 @@ class SoftwareClient(BaseClient):
21
21
  token: Authentication token
22
22
  timeout: Request timeout in seconds
23
23
  """
24
+ # Auto-append /api/v3 if missing
25
+ base_url = base_url.rstrip("/")
26
+ if not base_url.endswith("/api/v3"):
27
+ base_url = f"{base_url}/api/v3"
28
+
24
29
  super().__init__(base_url, token, timeout)
25
30
 
26
31
  # Catalog operations
@@ -609,10 +614,10 @@ class SoftwareClient(BaseClient):
609
614
  return self.put(f"reflection/{reflection_id}", data=reflection_data)
610
615
 
611
616
  def delete_reflection(self, reflection_id: str) -> None:
612
- """Delete a reflection.
613
-
614
- Args:
615
- reflection_id: Reflection ID
616
- """
617
+ """Delete a reflection."""
617
618
  return self.delete(f"reflection/{reflection_id}")
618
619
 
620
+ # Lineage operations
621
+ def get_catalog_graph(self, catalog_id: str) -> Dict[str, Any]:
622
+ """Get lineage graph for a catalog item."""
623
+ return self.get(f"catalog/{catalog_id}/graph")
@@ -319,3 +319,85 @@ def get_job_reflections(ctx, job_id: str) -> None:
319
319
  import traceback
320
320
  traceback.print_exc()
321
321
  raise click.Abort()
322
+ @job.command("analyze")
323
+ @click.argument("job_id")
324
+ @click.pass_context
325
+ def analyze_job(ctx, job_id: str) -> None:
326
+ """Analyze job performance.
327
+
328
+ Examples:
329
+ dremio job analyze <job-id>
330
+ """
331
+ try:
332
+ manager = ProfileManager()
333
+ profile_name = ctx.obj.profile_name
334
+ profile = manager.get_profile(profile_name)
335
+
336
+ if not profile:
337
+ console.print(f"[red]Profile '{profile_name}' not found[/red]")
338
+ return
339
+
340
+ client = create_client(profile)
341
+
342
+ with console.status(f"[bold green]Analyzing job {job_id}..."):
343
+ job = client.get_job(job_id)
344
+
345
+ # Basic info
346
+ state = job.get("jobState")
347
+ start = job.get("startTime")
348
+ end = job.get("endTime")
349
+
350
+ # Stats (structure varies between Cloud/Software versions)
351
+ # Safe access to stats
352
+ stats = job.get("stats", {})
353
+
354
+ # Provide insights
355
+ from rich.panel import Panel
356
+ from rich.columns import Columns
357
+
358
+ # Duration
359
+ duration = "N/A"
360
+ if start and end:
361
+ # Basic duration calc if not in stats (assuming epoch ms)
362
+ try:
363
+ diff = (int(end) - int(start)) / 1000.0
364
+ duration = f"{diff:.2f}s"
365
+ except:
366
+ pass
367
+
368
+ # Input/Output
369
+ input_bytes = stats.get("inputBytes", 0)
370
+ output_bytes = stats.get("outputBytes", 0)
371
+
372
+ # Simple insights
373
+ insights = []
374
+ if state != "COMPLETED":
375
+ insights.append(f"[red]Job did not complete ({state})[/red]")
376
+
377
+ input_records = stats.get("inputRecords", 0)
378
+ output_records = stats.get("outputRecords", 0)
379
+ if input_records > 0 and output_records > 0:
380
+ reduction = (1 - (output_records / input_records)) * 100
381
+ insights.append(f"[green]Data Reduction: {reduction:.1f}%[/green]")
382
+
383
+ # Display
384
+ console.print(Panel(f"""
385
+ [bold]Job ID[/bold]: {job_id}
386
+ [bold]State[/bold]: {state}
387
+ [bold]Duration[/bold]: {duration}
388
+ [bold]User[/bold]: {job.get('user', 'N/A')}
389
+ """, title="Job Summary", border_style="cyan"))
390
+
391
+ if insights:
392
+ console.print(Panel("\n".join(insights), title="Insights", border_style="yellow"))
393
+
394
+ # Raw stats table
395
+ from rich.table import Table
396
+ t = Table(title="Metrics", show_header=False)
397
+ t.add_row("Input Records", str(input_records))
398
+ t.add_row("Output Records", str(output_records))
399
+ t.add_row("Memory Used", f"{stats.get('memoryAllocation', 0)} bytes")
400
+ console.print(t)
401
+
402
+ except Exception as e:
403
+ console.print(f"[red]Analysis failed: {e}[/red]")
@@ -0,0 +1,88 @@
1
+ """Lineage management commands."""
2
+
3
+ import click
4
+ import json
5
+ from rich.console import Console
6
+ from rich.tree import Tree
7
+
8
+ from dremio_cli.config import ProfileManager
9
+ from dremio_cli.client.factory import create_client
10
+ from dremio_cli.formatters.json import format_as_json
11
+
12
+ console = Console()
13
+
14
+ @click.group()
15
+ def lineage():
16
+ """Lineage visualization operations."""
17
+ pass
18
+
19
+ @lineage.command("show")
20
+ @click.argument("catalog_id")
21
+ @click.option("--format", type=click.Choice(["tree", "json", "mermaid"], case_sensitive=False), default="tree", help="Output format")
22
+ @click.pass_context
23
+ def show_lineage(ctx, catalog_id: str, format: str):
24
+ """Show lineage graph for a dataset.
25
+
26
+ Examples:
27
+ dremio lineage show <catalog-id>
28
+ dremio lineage show <catalog-id> --format mermaid
29
+ """
30
+ try:
31
+ manager = ProfileManager()
32
+ profile_name = ctx.obj.profile_name
33
+ profile = manager.get_profile(profile_name)
34
+
35
+ if not profile:
36
+ console.print(f"[red]Profile '{profile_name}' not found[/red]")
37
+ return
38
+
39
+ client = create_client(profile)
40
+
41
+ try:
42
+ graph = client.get_catalog_graph(catalog_id)
43
+ except Exception as e:
44
+ console.print(f"[red]Error fetching lineage: {e}[/red]")
45
+ # Fallback or specific error handling (e.g. 404)
46
+ return
47
+
48
+ if format == "json":
49
+ console.print(format_as_json(graph))
50
+ return
51
+
52
+ # Process graph for Tree/Mermaid
53
+ # Graph structure usually: { "parents": [...], "children": [...] }
54
+ # Or detailed nodes/edges depending on API version.
55
+ # Assuming Dremio V3 graph response structure:
56
+ # { "parents": [ { "id": "...", "path": [...] } ], "children": [] }
57
+
58
+ # We focus on parents (upstream lineage)
59
+ parents = graph.get("parents", [])
60
+
61
+ if format == "mermaid":
62
+ click.echo("graph TD")
63
+ # Current node
64
+ click.echo(f' root["{catalog_id}"]')
65
+ click.echo(" style root fill:#f9f,stroke:#333,stroke-width:2px")
66
+
67
+ for parent in parents:
68
+ p_path = ".".join(parent.get("path", []))
69
+ p_id = parent.get("id", p_path) # fallback if id missing
70
+ # Sanitize ID for mermaid
71
+ safe_pid = p_id.replace("-", "_").replace(".", "_")
72
+
73
+ click.echo(f' {safe_pid}["{p_path}"] --> root')
74
+
75
+ else: # tree
76
+ tree = Tree(f"[bold cyan]{catalog_id}[/bold cyan]")
77
+ if parents:
78
+ p_branch = tree.add("Parents")
79
+ for parent in parents:
80
+ p_path = ".".join(parent.get("path", []))
81
+ p_branch.add(f"[green]{p_path}[/green]")
82
+ else:
83
+ tree.add("[italic]No parents found[/italic]")
84
+
85
+ console.print(tree)
86
+
87
+ except Exception as e:
88
+ console.print(f"[red]Error: {e}[/red]")
@@ -148,7 +148,26 @@ def execute_sql(ctx, query: str, sql_file: str, context: str, async_mode: bool,
148
148
  console.print("[yellow]Warning: --output-file with multiple queries will subscribe/overwite. Currently not fully supported for multi-statement.[/yellow]")
149
149
 
150
150
  output_path = Path(output_file)
151
- if output_path.suffix == '.json':
151
+
152
+ # Advanced Formats (Pandas required)
153
+ if output_path.suffix in ['.parquet', '.csv']:
154
+ try:
155
+ import pandas as pd
156
+ df = pd.DataFrame(rows)
157
+
158
+ if output_path.suffix == '.parquet':
159
+ df.to_parquet(output_path)
160
+ else: # csv
161
+ df.to_csv(output_path, index=False)
162
+
163
+ except ImportError:
164
+ console.print("[red]Error: 'pandas' and 'pyarrow' are required for Parquet/CSV export. Install them with pip.[/red]")
165
+ raise click.Abort()
166
+ except Exception as e:
167
+ console.print(f"[red]Export failed: {e}[/red]")
168
+ raise click.Abort()
169
+
170
+ elif output_path.suffix == '.json':
152
171
  output_path.write_text(json.dumps(results, indent=2))
153
172
  elif output_path.suffix in ['.yaml', '.yml']:
154
173
  import yaml