dremio-cli 1.5.0__tar.gz → 1.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. {dremio_cli-1.5.0/dremio_cli.egg-info → dremio_cli-1.6.0}/PKG-INFO +12 -2
  2. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/README.md +9 -1
  3. dremio_cli-1.6.0/docs/dac.md +60 -0
  4. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/jobs.md +13 -0
  5. dremio_cli-1.6.0/docs/lineage.md +25 -0
  6. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/sql.md +3 -1
  7. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/cli.py +4 -0
  8. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/cloud.py +6 -0
  9. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/software.py +5 -5
  10. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/job.py +82 -0
  11. dremio_cli-1.6.0/dremio_cli/commands/lineage.py +88 -0
  12. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/sql.py +20 -1
  13. dremio_cli-1.6.0/dremio_cli/commands/sync.py +51 -0
  14. dremio_cli-1.6.0/dremio_cli/dac/__init__.py +3 -0
  15. dremio_cli-1.6.0/dremio_cli/dac/config.py +42 -0
  16. dremio_cli-1.6.0/dremio_cli/dac/diff.py +26 -0
  17. dremio_cli-1.6.0/dremio_cli/dac/state.py +84 -0
  18. dremio_cli-1.6.0/dremio_cli/dac/sync.py +344 -0
  19. dremio_cli-1.6.0/dremio_cli/utils/console.py +4 -0
  20. {dremio_cli-1.5.0 → dremio_cli-1.6.0/dremio_cli.egg-info}/PKG-INFO +12 -2
  21. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli.egg-info/SOURCES.txt +9 -0
  22. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli.egg-info/requires.txt +2 -0
  23. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/pyproject.toml +3 -1
  24. dremio_cli-1.5.0/dremio_cli/commands/lineage.py +0 -9
  25. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/LICENSE +0 -0
  26. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/MANIFEST.in +0 -0
  27. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/completions/dremio-completion.bash +0 -0
  28. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/completions/dremio-completion.zsh +0 -0
  29. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/README.md +0 -0
  30. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/advanced-features.md +0 -0
  31. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/catalog.md +0 -0
  32. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/completion.md +0 -0
  33. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/favorites.md +0 -0
  34. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/grants.md +0 -0
  35. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/history.md +0 -0
  36. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/init.md +0 -0
  37. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/installation.md +0 -0
  38. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/monitor.md +0 -0
  39. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/profiles.md +0 -0
  40. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/quickstart.md +0 -0
  41. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/reflections.md +0 -0
  42. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/repl.md +0 -0
  43. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/roles.md +0 -0
  44. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/scripts.md +0 -0
  45. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/sources.md +0 -0
  46. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/spaces-folders.md +0 -0
  47. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/tables.md +0 -0
  48. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/tags-wiki.md +0 -0
  49. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/users.md +0 -0
  50. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/docs/views.md +0 -0
  51. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/__init__.py +0 -0
  52. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/__main__.py +0 -0
  53. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/__init__.py +0 -0
  54. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/auth.py +0 -0
  55. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/base.py +0 -0
  56. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/client/factory.py +0 -0
  57. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/__init__.py +0 -0
  58. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/catalog.py +0 -0
  59. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/completion.py +0 -0
  60. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/favorite.py +0 -0
  61. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/folder.py +0 -0
  62. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/grant.py +0 -0
  63. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/history.py +0 -0
  64. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/init.py +0 -0
  65. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/monitor.py +0 -0
  66. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/profile.py +0 -0
  67. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/reflection.py +0 -0
  68. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/repl.py +0 -0
  69. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/role.py +0 -0
  70. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/script.py +0 -0
  71. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/source.py +0 -0
  72. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/space.py +0 -0
  73. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/table.py +0 -0
  74. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/tag.py +0 -0
  75. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/udf.py +0 -0
  76. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/user.py +0 -0
  77. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/view.py +0 -0
  78. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/commands/wiki.py +0 -0
  79. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/config.py +0 -0
  80. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/env.py +0 -0
  81. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/formatters/__init__.py +0 -0
  82. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/formatters/json.py +0 -0
  83. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/formatters/table.py +0 -0
  84. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/formatters/yaml.py +0 -0
  85. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/history.py +0 -0
  86. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/utils/__init__.py +0 -0
  87. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/utils/exceptions.py +0 -0
  88. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/utils/helpers.py +0 -0
  89. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli/utils/validators.py +0 -0
  90. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli.egg-info/dependency_links.txt +0 -0
  91. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli.egg-info/entry_points.txt +0 -0
  92. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/dremio_cli.egg-info/top_level.txt +0 -0
  93. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/setup.cfg +0 -0
  94. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/setup.py +0 -0
  95. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/__init__.py +0 -0
  96. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/conftest.py +0 -0
  97. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_config.py +0 -0
  98. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_grant_commands.py +0 -0
  99. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_history_favorite.py +0 -0
  100. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_integration.py +0 -0
  101. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_job_commands.py +0 -0
  102. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_multi_sql_execution.py +0 -0
  103. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_reflection_commands.py +0 -0
  104. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_role_commands.py +0 -0
  105. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_script_commands.py +0 -0
  106. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_source_commands.py +0 -0
  107. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_space_folder_commands.py +0 -0
  108. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_sql_commands.py +0 -0
  109. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_table_commands.py +0 -0
  110. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_tag_wiki_commands.py +0 -0
  111. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_user_commands.py +0 -0
  112. {dremio_cli-1.5.0 → dremio_cli-1.6.0}/tests/test_view_commands.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dremio-cli
3
- Version: 1.5.0
3
+ Version: 1.6.0
4
4
  Summary: A comprehensive command-line interface for Dremio Cloud and Dremio Software
5
5
  Home-page: https://github.com/your-org/dremio-cli
6
6
  Author: Your Name
@@ -38,6 +38,8 @@ Requires-Dist: python-dotenv>=1.0.0
38
38
  Requires-Dist: pyyaml>=6.0.0
39
39
  Requires-Dist: prompt_toolkit>=3.0.0
40
40
  Requires-Dist: textual>=0.40.0
41
+ Requires-Dist: pandas>=2.0.0
42
+ Requires-Dist: pyarrow>=12.0.0
41
43
  Provides-Extra: dev
42
44
  Requires-Dist: pytest>=7.0.0; extra == "dev"
43
45
  Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
@@ -88,7 +90,10 @@ dremio view create --path "Analytics.summary" \
88
90
 
89
91
  ## 📚 Documentation
90
92
 
91
- **[Complete Documentation →](docs/README.md)**
93
+ **[Complete Documentation (GitHub) →](https://github.com/developer-advocacy-dremio/dremio-python-cli/tree/main/dremio-cli/docs)**
94
+
95
+ *(Use the link above if browsing on PyPI)*
96
+
92
97
 
93
98
  ### Core Guides
94
99
 
@@ -118,6 +123,11 @@ dremio view create --path "Analytics.summary" \
118
123
 
119
124
  - **[Favorites](docs/favorites.md)** - Save frequently used queries
120
125
  - **[History](docs/history.md)** - View and re-run commands
126
+ - **[REPL](docs/repl.md)** - Interactive SQL Shell (Enhanced)
127
+ - **[Monitor](docs/monitor.md)** - Real-time System Monitor
128
+ - **[Init](docs/init.md)** - Interactive Configuration Wizard
129
+ - **[Completion](docs/completion.md)** - Shell Completion Scripts
130
+ - **[Lineage](docs/lineage.md)** - Visualize dataset dependencies
121
131
 
122
132
  ## 🎯 Key Features
123
133
 
@@ -37,7 +37,10 @@ dremio view create --path "Analytics.summary" \
37
37
 
38
38
  ## 📚 Documentation
39
39
 
40
- **[Complete Documentation →](docs/README.md)**
40
+ **[Complete Documentation (GitHub) →](https://github.com/developer-advocacy-dremio/dremio-python-cli/tree/main/dremio-cli/docs)**
41
+
42
+ *(Use the link above if browsing on PyPI)*
43
+
41
44
 
42
45
  ### Core Guides
43
46
 
@@ -67,6 +70,11 @@ dremio view create --path "Analytics.summary" \
67
70
 
68
71
  - **[Favorites](docs/favorites.md)** - Save frequently used queries
69
72
  - **[History](docs/history.md)** - View and re-run commands
73
+ - **[REPL](docs/repl.md)** - Interactive SQL Shell (Enhanced)
74
+ - **[Monitor](docs/monitor.md)** - Real-time System Monitor
75
+ - **[Init](docs/init.md)** - Interactive Configuration Wizard
76
+ - **[Completion](docs/completion.md)** - Shell Completion Scripts
77
+ - **[Lineage](docs/lineage.md)** - Visualize dataset dependencies
70
78
 
71
79
  ## 🎯 Key Features
72
80
 
@@ -0,0 +1,60 @@
1
+ # Dremio-as-Code (GitOps)
2
+
3
+ Dremio-as-Code (DAC) allows you to manage your Dremio catalog (Spaces, Folders, Views) using local files, enabling GitOps workflows.
4
+
5
+ ## Quick Start
6
+
7
+ ### 1. Initialize Configuration
8
+ Create a `dremio.yaml` file in your project root:
9
+
10
+ ```yaml
11
+ version: "1.0"
12
+ scope:
13
+ path: "dremio-catalog.finance" # The Dremio path to sync
14
+ type: "SPACE" # SPACE or ICEBERGCATALOG
15
+ ignore:
16
+ - "*.tmp"
17
+ ```
18
+
19
+ ### 2. Pull State
20
+ Capture the current state of your Dremio space into local files.
21
+
22
+ ```bash
23
+ dremio sync pull
24
+ ```
25
+
26
+ Returns a directory structure mirroring Dremio:
27
+ - `finance/`
28
+ - `monthly_report.sql`
29
+ - `monthly_report.yaml`
30
+
31
+ ### 3. Make Changes
32
+ Edit a view's SQL locally or create a new file pair (`.sql` and `.yaml`).
33
+
34
+ **new_view.sql**:
35
+ ```sql
36
+ SELECT * FROM parent.table
37
+ ```
38
+
39
+ **new_view.yaml**:
40
+ ```yaml
41
+ type: VIEW
42
+ path: ["dremio-catalog", "finance", "new_view"]
43
+ sql_file: "new_view.sql"
44
+ context: []
45
+ ```
46
+
47
+ ### 4. Push Changes
48
+ Apply your local changes back to Dremio.
49
+
50
+ ```bash
51
+ dremio sync push
52
+ ```
53
+
54
+ ## Concepts
55
+
56
+ - **Scope**: Limits the sync to a specific subtree to support multi-team environments.
57
+ - **State File**: `.dremio_state.json` tracks the last known state to enable efficient updates and deletes. **Do not edit this file manually.**
58
+ - **File Format**:
59
+ - **Context**: List of path parts to set query context.
60
+ - **Reflections**: (Coming Soon) Defined in YAML.
@@ -60,6 +60,19 @@ dremio --output json job get 16b2c9cd-a920-952b-b162-2280c9059d00
60
60
  dremio --verbose job get 16b2c9cd-a920-952b-b162-2280c9059d00
61
61
  ```
62
62
 
63
+ ### Analyze Job
64
+
65
+ Analyze the performance of a job to identify bottlenecks.
66
+
67
+ ```bash
68
+ dremio job analyze <JOB_ID>
69
+ ```
70
+
71
+ **Output:**
72
+ - Job duration and state
73
+ - Data reduction ratio (input vs output records)
74
+ - Insights on performance metrics
75
+
63
76
  ### Get Job Results
64
77
 
65
78
  Retrieve the results of a completed job.
@@ -0,0 +1,25 @@
1
+ # Lineage Visualization
2
+
3
+ Visualize the dependencies between datasets.
4
+
5
+ ## Commands
6
+
7
+ ### Show Lineage
8
+
9
+ Show the upstream parents of a dataset.
10
+
11
+ ```bash
12
+ dremio lineage show <CATALOG_ID> [OPTIONS]
13
+ ```
14
+
15
+ **Options:**
16
+ - `--format [tree|json|mermaid]` - Output format (Default: tree)
17
+
18
+ **Examples:**
19
+ ```bash
20
+ # Tree view (Terminal)
21
+ dremio lineage show dremio-catalog.space.view
22
+
23
+ # Mermaid Graph (for markdown)
24
+ dremio lineage show dremio-catalog.space.view --format mermaid
25
+ ```
@@ -20,7 +20,9 @@ dremio sql execute --file <FILE> [OPTIONS]
20
20
  - `--file PATH` - Execute SQL from file
21
21
  - `--context TEXT` - Query context (comma-separated path)
22
22
  - `--async` - Execute asynchronously (return job ID immediately). Default behavior waits for results.
23
- - `--output-file PATH` - Save results to file
23
+ - `--output-file PATH` - Save results to file (supports .json, .yaml, .csv, .parquet)
24
+
25
+ **Note:** `.csv` and `.parquet` export require `pandas` and `pyarrow` installed.
24
26
 
25
27
  **Examples:**
26
28
 
@@ -30,6 +30,8 @@ from dremio_cli.commands import (
30
30
  repl,
31
31
  monitor,
32
32
  completion,
33
+ lineage,
34
+ sync,
33
35
  )
34
36
 
35
37
 
@@ -102,6 +104,8 @@ cli.add_command(init.init_command)
102
104
  cli.add_command(repl.repl_command)
103
105
  cli.add_command(monitor.monitor_command)
104
106
  cli.add_command(completion.completion_command)
107
+ cli.add_command(lineage.lineage)
108
+ cli.add_command(sync.sync)
105
109
 
106
110
 
107
111
 
@@ -363,3 +363,9 @@ class CloudClient(BaseClient):
363
363
  """Delete a script."""
364
364
  return self.delete(self._project_endpoint(f"scripts/{script_id}"))
365
365
 
366
+ # Lineage operations
367
+ def get_catalog_graph(self, catalog_id: str) -> Dict[str, Any]:
368
+ """Get lineage graph for a catalog item."""
369
+ # Cloud API usually follows similar pattern or project endpoint
370
+ return self.get(self._project_endpoint(f"catalog/{catalog_id}/graph"))
371
+
@@ -609,10 +609,10 @@ class SoftwareClient(BaseClient):
609
609
  return self.put(f"reflection/{reflection_id}", data=reflection_data)
610
610
 
611
611
  def delete_reflection(self, reflection_id: str) -> None:
612
- """Delete a reflection.
613
-
614
- Args:
615
- reflection_id: Reflection ID
616
- """
612
+ """Delete a reflection."""
617
613
  return self.delete(f"reflection/{reflection_id}")
618
614
 
615
+ # Lineage operations
616
+ def get_catalog_graph(self, catalog_id: str) -> Dict[str, Any]:
617
+ """Get lineage graph for a catalog item."""
618
+ return self.get(f"catalog/{catalog_id}/graph")
@@ -319,3 +319,85 @@ def get_job_reflections(ctx, job_id: str) -> None:
319
319
  import traceback
320
320
  traceback.print_exc()
321
321
  raise click.Abort()
322
+ @job.command("analyze")
323
+ @click.argument("job_id")
324
+ @click.pass_context
325
+ def analyze_job(ctx, job_id: str) -> None:
326
+ """Analyze job performance.
327
+
328
+ Examples:
329
+ dremio job analyze <job-id>
330
+ """
331
+ try:
332
+ manager = ProfileManager()
333
+ profile_name = ctx.obj.profile_name
334
+ profile = manager.get_profile(profile_name)
335
+
336
+ if not profile:
337
+ console.print(f"[red]Profile '{profile_name}' not found[/red]")
338
+ return
339
+
340
+ client = create_client(profile)
341
+
342
+ with console.status(f"[bold green]Analyzing job {job_id}..."):
343
+ job = client.get_job(job_id)
344
+
345
+ # Basic info
346
+ state = job.get("jobState")
347
+ start = job.get("startTime")
348
+ end = job.get("endTime")
349
+
350
+ # Stats (structure varies between Cloud/Software versions)
351
+ # Safe access to stats
352
+ stats = job.get("stats", {})
353
+
354
+ # Provide insights
355
+ from rich.panel import Panel
356
+ from rich.columns import Columns
357
+
358
+ # Duration
359
+ duration = "N/A"
360
+ if start and end:
361
+ # Basic duration calc if not in stats (assuming epoch ms)
362
+ try:
363
+ diff = (int(end) - int(start)) / 1000.0
364
+ duration = f"{diff:.2f}s"
365
+ except:
366
+ pass
367
+
368
+ # Input/Output
369
+ input_bytes = stats.get("inputBytes", 0)
370
+ output_bytes = stats.get("outputBytes", 0)
371
+
372
+ # Simple insights
373
+ insights = []
374
+ if state != "COMPLETED":
375
+ insights.append(f"[red]Job did not complete ({state})[/red]")
376
+
377
+ input_records = stats.get("inputRecords", 0)
378
+ output_records = stats.get("outputRecords", 0)
379
+ if input_records > 0 and output_records > 0:
380
+ reduction = (1 - (output_records / input_records)) * 100
381
+ insights.append(f"[green]Data Reduction: {reduction:.1f}%[/green]")
382
+
383
+ # Display
384
+ console.print(Panel(f"""
385
+ [bold]Job ID[/bold]: {job_id}
386
+ [bold]State[/bold]: {state}
387
+ [bold]Duration[/bold]: {duration}
388
+ [bold]User[/bold]: {job.get('user', 'N/A')}
389
+ """, title="Job Summary", border_style="cyan"))
390
+
391
+ if insights:
392
+ console.print(Panel("\n".join(insights), title="Insights", border_style="yellow"))
393
+
394
+ # Raw stats table
395
+ from rich.table import Table
396
+ t = Table(title="Metrics", show_header=False)
397
+ t.add_row("Input Records", str(input_records))
398
+ t.add_row("Output Records", str(output_records))
399
+ t.add_row("Memory Used", f"{stats.get('memoryAllocation', 0)} bytes")
400
+ console.print(t)
401
+
402
+ except Exception as e:
403
+ console.print(f"[red]Analysis failed: {e}[/red]")
@@ -0,0 +1,88 @@
1
+ """Lineage management commands."""
2
+
3
+ import click
4
+ import json
5
+ from rich.console import Console
6
+ from rich.tree import Tree
7
+
8
+ from dremio_cli.config import ProfileManager
9
+ from dremio_cli.client.factory import create_client
10
+ from dremio_cli.formatters.json import format_as_json
11
+
12
+ console = Console()
13
+
14
+ @click.group()
15
+ def lineage():
16
+ """Lineage visualization operations."""
17
+ pass
18
+
19
+ @lineage.command("show")
20
+ @click.argument("catalog_id")
21
+ @click.option("--format", type=click.Choice(["tree", "json", "mermaid"], case_sensitive=False), default="tree", help="Output format")
22
+ @click.pass_context
23
+ def show_lineage(ctx, catalog_id: str, format: str):
24
+ """Show lineage graph for a dataset.
25
+
26
+ Examples:
27
+ dremio lineage show <catalog-id>
28
+ dremio lineage show <catalog-id> --format mermaid
29
+ """
30
+ try:
31
+ manager = ProfileManager()
32
+ profile_name = ctx.obj.profile_name
33
+ profile = manager.get_profile(profile_name)
34
+
35
+ if not profile:
36
+ console.print(f"[red]Profile '{profile_name}' not found[/red]")
37
+ return
38
+
39
+ client = create_client(profile)
40
+
41
+ try:
42
+ graph = client.get_catalog_graph(catalog_id)
43
+ except Exception as e:
44
+ console.print(f"[red]Error fetching lineage: {e}[/red]")
45
+ # Fallback or specific error handling (e.g. 404)
46
+ return
47
+
48
+ if format == "json":
49
+ console.print(format_as_json(graph))
50
+ return
51
+
52
+ # Process graph for Tree/Mermaid
53
+ # Graph structure usually: { "parents": [...], "children": [...] }
54
+ # Or detailed nodes/edges depending on API version.
55
+ # Assuming Dremio V3 graph response structure:
56
+ # { "parents": [ { "id": "...", "path": [...] } ], "children": [] }
57
+
58
+ # We focus on parents (upstream lineage)
59
+ parents = graph.get("parents", [])
60
+
61
+ if format == "mermaid":
62
+ click.echo("graph TD")
63
+ # Current node
64
+ click.echo(f' root["{catalog_id}"]')
65
+ click.echo(" style root fill:#f9f,stroke:#333,stroke-width:2px")
66
+
67
+ for parent in parents:
68
+ p_path = ".".join(parent.get("path", []))
69
+ p_id = parent.get("id", p_path) # fallback if id missing
70
+ # Sanitize ID for mermaid
71
+ safe_pid = p_id.replace("-", "_").replace(".", "_")
72
+
73
+ click.echo(f' {safe_pid}["{p_path}"] --> root')
74
+
75
+ else: # tree
76
+ tree = Tree(f"[bold cyan]{catalog_id}[/bold cyan]")
77
+ if parents:
78
+ p_branch = tree.add("Parents")
79
+ for parent in parents:
80
+ p_path = ".".join(parent.get("path", []))
81
+ p_branch.add(f"[green]{p_path}[/green]")
82
+ else:
83
+ tree.add("[italic]No parents found[/italic]")
84
+
85
+ console.print(tree)
86
+
87
+ except Exception as e:
88
+ console.print(f"[red]Error: {e}[/red]")
@@ -148,7 +148,26 @@ def execute_sql(ctx, query: str, sql_file: str, context: str, async_mode: bool,
148
148
  console.print("[yellow]Warning: --output-file with multiple queries will subscribe/overwite. Currently not fully supported for multi-statement.[/yellow]")
149
149
 
150
150
  output_path = Path(output_file)
151
- if output_path.suffix == '.json':
151
+
152
+ # Advanced Formats (Pandas required)
153
+ if output_path.suffix in ['.parquet', '.csv']:
154
+ try:
155
+ import pandas as pd
156
+ df = pd.DataFrame(rows)
157
+
158
+ if output_path.suffix == '.parquet':
159
+ df.to_parquet(output_path)
160
+ else: # csv
161
+ df.to_csv(output_path, index=False)
162
+
163
+ except ImportError:
164
+ console.print("[red]Error: 'pandas' and 'pyarrow' are required for Parquet/CSV export. Install them with pip.[/red]")
165
+ raise click.Abort()
166
+ except Exception as e:
167
+ console.print(f"[red]Export failed: {e}[/red]")
168
+ raise click.Abort()
169
+
170
+ elif output_path.suffix == '.json':
152
171
  output_path.write_text(json.dumps(results, indent=2))
153
172
  elif output_path.suffix in ['.yaml', '.yml']:
154
173
  import yaml
@@ -0,0 +1,51 @@
1
+
2
+ import os
3
+ import click
4
+ from dremio_cli.utils.console import console
5
+ from dremio_cli.dac.config import DremioConfig
6
+ from dremio_cli.dac.sync import DremioSync
7
+
8
+ @click.group(help="Dremio-as-Code synchronization.")
9
+ def sync():
10
+ pass
11
+
12
+ @sync.command("pull")
13
+ @click.option("--config", default="dremio.yaml", help="Path to dremio.yaml config file.")
14
+ @click.pass_context
15
+ def pull(ctx, config):
16
+ """Pull state from Dremio to local filesystem."""
17
+ try:
18
+ if not os.path.exists(config):
19
+ console.print(f"[red]Config file '{config}' not found. Please create a dremio.yaml first.[/red]")
20
+ return
21
+
22
+ conf = DremioConfig.load(config)
23
+ client = ctx.obj
24
+
25
+ syncer = DremioSync(conf, client, os.path.dirname(os.path.abspath(config)))
26
+ syncer.pull()
27
+
28
+ except Exception as e:
29
+ console.print(f"[red]Pull failed: {e}[/red]")
30
+ # import traceback
31
+ # traceback.print_exc()
32
+
33
+ @sync.command("push")
34
+ @click.option("--config", default="dremio.yaml", help="Path to dremio.yaml config file.")
35
+ @click.option("--dry-run", is_flag=True, help="Show changes without applying them.")
36
+ @click.pass_context
37
+ def push(ctx, config, dry_run):
38
+ """Push local state to Dremio."""
39
+ try:
40
+ if not os.path.exists(config):
41
+ console.print(f"[red]Config file '{config}' not found.[/red]")
42
+ return
43
+
44
+ conf = DremioConfig.load(config)
45
+ client = ctx.obj
46
+
47
+ syncer = DremioSync(conf, client, os.path.dirname(os.path.abspath(config)))
48
+ syncer.push(dry_run=dry_run)
49
+
50
+ except Exception as e:
51
+ console.print(f"[red]Push failed: {e}[/red]")
@@ -0,0 +1,3 @@
1
+ """
2
+ Dremio-as-Code (DAC) module.
3
+ """
@@ -0,0 +1,42 @@
1
+
2
+ import os
3
+ import yaml
4
+ from dataclasses import dataclass
5
+ from typing import List, Optional
6
+
7
+ @dataclass
8
+ class Scope:
9
+ path: str
10
+ type: str # SPACE, ICEBERGCATALOG, SOURCE
11
+
12
+ @dataclass
13
+ class DremioConfig:
14
+ version: str
15
+ scope: Scope
16
+ ignore: List[str]
17
+
18
+ @classmethod
19
+ def load(cls, path: str = "dremio.yaml") -> 'DremioConfig':
20
+ if not os.path.exists(path):
21
+ raise FileNotFoundError(f"Config file {path} not found.")
22
+
23
+ with open(path, "r") as f:
24
+ data = yaml.safe_load(f)
25
+
26
+ scope_data = data.get("scope", {})
27
+ if not scope_data.get("path") or not scope_data.get("type"):
28
+ raise ValueError("dremio.yaml must define scope.path and scope.type")
29
+
30
+ # Validate type
31
+ valid_types = ["SPACE", "ICEBERGCATALOG", "SOURCE"]
32
+ if scope_data["type"] not in valid_types:
33
+ raise ValueError(f"Invalid scope.type: {scope_data['type']}. Must be one of {valid_types}")
34
+
35
+ return cls(
36
+ version=data.get("version", "1.0"),
37
+ scope=Scope(
38
+ path=scope_data["path"],
39
+ type=scope_data["type"]
40
+ ),
41
+ ignore=data.get("ignore", [])
42
+ )
@@ -0,0 +1,26 @@
1
+
2
+ from typing import List, Dict, Any
3
+ from dataclasses import dataclass
4
+
5
+ @dataclass
6
+ class DiffItem:
7
+ action: str # CREATE, UPDATE, DELETE, NOOP
8
+ path: List[str]
9
+ type: str
10
+ reason: str
11
+ local_data: Dict[str, Any] = None
12
+
13
+ class Differ:
14
+ def __init__(self, local_state: 'LocalState', remote_state: Dict[str, Any]):
15
+ """
16
+ local_state: The LocalState object managing the .dremio_state.json
17
+ remote_state: A dictionary representing the current structure on Dremio side (fetched via API)
18
+ """
19
+ self.local = local_state
20
+ self.remote = remote_state
21
+
22
+ # TODO: Implement diff logic in the Push implementation phase
23
+ # This serves as a placeholder for the module structure.
24
+ def compare(self) -> List[DiffItem]:
25
+ diffs = []
26
+ return diffs
@@ -0,0 +1,84 @@
1
+
2
+ import os
3
+ import json
4
+ import yaml
5
+ from typing import Dict, Any, List, Optional
6
+ from dataclasses import dataclass, asdict
7
+
8
+ @dataclass
9
+ class ResourceState:
10
+ id: str
11
+ path: List[str]
12
+ type: str # DATASET, FOLDER, SPACE, SOURCE, VIEW (Specific)
13
+ hash: str # Content hash for change detection
14
+ metadata: Dict[str, Any]
15
+
16
+ class LocalState:
17
+ def __init__(self, root_path: str):
18
+ self.root_path = root_path
19
+ self.state_file = os.path.join(root_path, ".dremio_state.json")
20
+ self.resources: Dict[str, ResourceState] = {} # Keyed by path string "space.folder.view"
21
+
22
+ def load(self):
23
+ """Load state from .dremio_state.json"""
24
+ if os.path.exists(self.state_file):
25
+ try:
26
+ with open(self.state_file, 'r') as f:
27
+ data = json.load(f)
28
+ for key, item in data.items():
29
+ self.resources[key] = ResourceState(**item)
30
+ except Exception as e:
31
+ print(f"Warning: Could not load state file: {e}")
32
+
33
+ def save(self):
34
+ """Save state to .dremio_state.json"""
35
+ data = {k: asdict(v) for k, v in self.resources.items()}
36
+ with open(self.state_file, 'w') as f:
37
+ json.dump(data, f, indent=2)
38
+
39
+ def update_resource(self, path_list: List[str], type: str, id: str, hash: str, metadata: Dict[str, Any]):
40
+ key = ".".join(path_list)
41
+ self.resources[key] = ResourceState(
42
+ id=id,
43
+ path=path_list,
44
+ type=type,
45
+ hash=hash,
46
+ metadata=metadata
47
+ )
48
+
49
+ def get_resource(self, path_list: List[str]) -> Optional[ResourceState]:
50
+ key = ".".join(path_list)
51
+ return self.resources.get(key)
52
+
53
+ def remove_resource(self, path_list: List[str]):
54
+ key = ".".join(path_list)
55
+ if key in self.resources:
56
+ del self.resources[key]
57
+
58
+ def scan_filesystem(self) -> Dict[str, Dict[str, Any]]:
59
+ """
60
+ Scans the root_dir for .yaml and .sql files to build a picture of local state.
61
+ Returns a dict of resource definitions found on disk.
62
+ """
63
+ local_resources = {}
64
+ for root, dirs, files in os.walk(self.root_path):
65
+ # skipping .git, .dremio_state.json, etc is handled by not parsing them
66
+ for file in files:
67
+ if file.endswith(".yaml") and file != "dremio.yaml":
68
+ full_path = os.path.join(root, file)
69
+ try:
70
+ with open(full_path, 'r') as f:
71
+ data = yaml.safe_load(f)
72
+ # We expect 'type' and 'path' in the yaml
73
+ if data and "type" in data and "path" in data:
74
+ path_key = ".".join(data["path"])
75
+ local_resources[path_key] = data
76
+ # If there is a sql file, read it
77
+ if "sql_file" in data:
78
+ sql_path = os.path.join(root, data["sql_file"])
79
+ if os.path.exists(sql_path):
80
+ with open(sql_path, 'r') as sql_f:
81
+ data["sql_content"] = sql_f.read()
82
+ except Exception as e:
83
+ print(f"Error reading {full_path}: {e}")
84
+ return local_resources