dremio-cli 1.6.0__tar.gz → 1.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. {dremio_cli-1.6.0/dremio_cli.egg-info → dremio_cli-1.8.0}/PKG-INFO +11 -1
  2. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/README.md +10 -0
  3. dremio_cli-1.8.0/docs/dac.md +128 -0
  4. dremio_cli-1.8.0/docs/dac_governance.md +88 -0
  5. dremio_cli-1.8.0/docs/dac_reflections.md +63 -0
  6. dremio_cli-1.8.0/docs/dac_sources.md +80 -0
  7. dremio_cli-1.8.0/docs/dac_tables.md +78 -0
  8. dremio_cli-1.8.0/docs/dac_validations.md +54 -0
  9. dremio_cli-1.8.0/docs/profiles.md +139 -0
  10. dremio_cli-1.8.0/docs/tui.md +44 -0
  11. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/cli.py +2 -0
  12. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/client/auth.py +6 -1
  13. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/client/software.py +5 -0
  14. dremio_cli-1.8.0/dremio_cli/commands/ui.py +20 -0
  15. dremio_cli-1.8.0/dremio_cli/dac/graph.py +69 -0
  16. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/dac/state.py +47 -12
  17. dremio_cli-1.8.0/dremio_cli/dac/sync.py +863 -0
  18. dremio_cli-1.8.0/dremio_cli/tui/catalog/__init__.py +1 -0
  19. dremio_cli-1.8.0/dremio_cli/tui/catalog/app.py +34 -0
  20. dremio_cli-1.8.0/dremio_cli/tui/catalog/screens.py +117 -0
  21. dremio_cli-1.8.0/dremio_cli/tui/catalog/widgets.py +180 -0
  22. {dremio_cli-1.6.0 → dremio_cli-1.8.0/dremio_cli.egg-info}/PKG-INFO +11 -1
  23. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli.egg-info/SOURCES.txt +12 -0
  24. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/pyproject.toml +1 -1
  25. dremio_cli-1.6.0/docs/dac.md +0 -60
  26. dremio_cli-1.6.0/docs/profiles.md +0 -378
  27. dremio_cli-1.6.0/dremio_cli/dac/sync.py +0 -344
  28. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/LICENSE +0 -0
  29. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/MANIFEST.in +0 -0
  30. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/completions/dremio-completion.bash +0 -0
  31. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/completions/dremio-completion.zsh +0 -0
  32. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/README.md +0 -0
  33. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/advanced-features.md +0 -0
  34. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/catalog.md +0 -0
  35. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/completion.md +0 -0
  36. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/favorites.md +0 -0
  37. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/grants.md +0 -0
  38. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/history.md +0 -0
  39. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/init.md +0 -0
  40. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/installation.md +0 -0
  41. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/jobs.md +0 -0
  42. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/lineage.md +0 -0
  43. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/monitor.md +0 -0
  44. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/quickstart.md +0 -0
  45. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/reflections.md +0 -0
  46. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/repl.md +0 -0
  47. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/roles.md +0 -0
  48. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/scripts.md +0 -0
  49. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/sources.md +0 -0
  50. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/spaces-folders.md +0 -0
  51. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/sql.md +0 -0
  52. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/tables.md +0 -0
  53. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/tags-wiki.md +0 -0
  54. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/users.md +0 -0
  55. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/docs/views.md +0 -0
  56. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/__init__.py +0 -0
  57. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/__main__.py +0 -0
  58. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/client/__init__.py +0 -0
  59. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/client/base.py +0 -0
  60. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/client/cloud.py +0 -0
  61. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/client/factory.py +0 -0
  62. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/__init__.py +0 -0
  63. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/catalog.py +0 -0
  64. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/completion.py +0 -0
  65. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/favorite.py +0 -0
  66. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/folder.py +0 -0
  67. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/grant.py +0 -0
  68. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/history.py +0 -0
  69. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/init.py +0 -0
  70. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/job.py +0 -0
  71. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/lineage.py +0 -0
  72. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/monitor.py +0 -0
  73. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/profile.py +0 -0
  74. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/reflection.py +0 -0
  75. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/repl.py +0 -0
  76. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/role.py +0 -0
  77. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/script.py +0 -0
  78. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/source.py +0 -0
  79. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/space.py +0 -0
  80. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/sql.py +0 -0
  81. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/sync.py +0 -0
  82. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/table.py +0 -0
  83. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/tag.py +0 -0
  84. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/udf.py +0 -0
  85. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/user.py +0 -0
  86. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/view.py +0 -0
  87. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/commands/wiki.py +0 -0
  88. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/config.py +0 -0
  89. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/dac/__init__.py +0 -0
  90. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/dac/config.py +0 -0
  91. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/dac/diff.py +0 -0
  92. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/env.py +0 -0
  93. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/formatters/__init__.py +0 -0
  94. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/formatters/json.py +0 -0
  95. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/formatters/table.py +0 -0
  96. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/formatters/yaml.py +0 -0
  97. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/history.py +0 -0
  98. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/utils/__init__.py +0 -0
  99. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/utils/console.py +0 -0
  100. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/utils/exceptions.py +0 -0
  101. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/utils/helpers.py +0 -0
  102. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli/utils/validators.py +0 -0
  103. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli.egg-info/dependency_links.txt +0 -0
  104. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli.egg-info/entry_points.txt +0 -0
  105. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli.egg-info/requires.txt +0 -0
  106. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/dremio_cli.egg-info/top_level.txt +0 -0
  107. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/setup.cfg +0 -0
  108. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/setup.py +0 -0
  109. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/__init__.py +0 -0
  110. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/conftest.py +0 -0
  111. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_config.py +0 -0
  112. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_grant_commands.py +0 -0
  113. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_history_favorite.py +0 -0
  114. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_integration.py +0 -0
  115. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_job_commands.py +0 -0
  116. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_multi_sql_execution.py +0 -0
  117. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_reflection_commands.py +0 -0
  118. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_role_commands.py +0 -0
  119. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_script_commands.py +0 -0
  120. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_source_commands.py +0 -0
  121. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_space_folder_commands.py +0 -0
  122. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_sql_commands.py +0 -0
  123. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_table_commands.py +0 -0
  124. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_tag_wiki_commands.py +0 -0
  125. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_user_commands.py +0 -0
  126. {dremio_cli-1.6.0 → dremio_cli-1.8.0}/tests/test_view_commands.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dremio-cli
3
- Version: 1.6.0
3
+ Version: 1.8.0
4
4
  Summary: A comprehensive command-line interface for Dremio Cloud and Dremio Software
5
5
  Home-page: https://github.com/your-org/dremio-cli
6
6
  Author: Your Name
@@ -119,12 +119,22 @@ dremio view create --path "Analytics.summary" \
119
119
  - **[Reflections](docs/reflections.md)** - Manage reflections
120
120
  - **[Scripts](docs/scripts.md)** - Manage scripts (Cloud only)
121
121
 
122
+ ### Dremio-as-Code (GitOps)
123
+
124
+ - **[Overview](docs/dac.md)** - Getting Started with DAC
125
+ - **[Sources](docs/dac_sources.md)** - Manage Sources
126
+ - **[Tables](docs/dac_tables.md)** - Manage Physical Tables
127
+ - **[Validations](docs/dac_validations.md)** - Data Quality Checks
128
+ - **[Reflections](docs/dac_reflections.md)** - Manage Reflections
129
+ - **[Governance](docs/dac_governance.md)** - Managing Access Control
130
+
122
131
  ### Productivity
123
132
 
124
133
  - **[Favorites](docs/favorites.md)** - Save frequently used queries
125
134
  - **[History](docs/history.md)** - View and re-run commands
126
135
  - **[REPL](docs/repl.md)** - Interactive SQL Shell (Enhanced)
127
136
  - **[Monitor](docs/monitor.md)** - Real-time System Monitor
137
+ - **[Catalog Explorer](docs/tui.md)** - Interactive Catalog TUI
128
138
  - **[Init](docs/init.md)** - Interactive Configuration Wizard
129
139
  - **[Completion](docs/completion.md)** - Shell Completion Scripts
130
140
  - **[Lineage](docs/lineage.md)** - Visualize dataset dependencies
@@ -66,12 +66,22 @@ dremio view create --path "Analytics.summary" \
66
66
  - **[Reflections](docs/reflections.md)** - Manage reflections
67
67
  - **[Scripts](docs/scripts.md)** - Manage scripts (Cloud only)
68
68
 
69
+ ### Dremio-as-Code (GitOps)
70
+
71
+ - **[Overview](docs/dac.md)** - Getting Started with DAC
72
+ - **[Sources](docs/dac_sources.md)** - Manage Sources
73
+ - **[Tables](docs/dac_tables.md)** - Manage Physical Tables
74
+ - **[Validations](docs/dac_validations.md)** - Data Quality Checks
75
+ - **[Reflections](docs/dac_reflections.md)** - Manage Reflections
76
+ - **[Governance](docs/dac_governance.md)** - Managing Access Control
77
+
69
78
  ### Productivity
70
79
 
71
80
  - **[Favorites](docs/favorites.md)** - Save frequently used queries
72
81
  - **[History](docs/history.md)** - View and re-run commands
73
82
  - **[REPL](docs/repl.md)** - Interactive SQL Shell (Enhanced)
74
83
  - **[Monitor](docs/monitor.md)** - Real-time System Monitor
84
+ - **[Catalog Explorer](docs/tui.md)** - Interactive Catalog TUI
75
85
  - **[Init](docs/init.md)** - Interactive Configuration Wizard
76
86
  - **[Completion](docs/completion.md)** - Shell Completion Scripts
77
87
  - **[Lineage](docs/lineage.md)** - Visualize dataset dependencies
@@ -0,0 +1,128 @@
1
+ # Dremio-as-Code (GitOps)
2
+
3
+ Dremio-as-Code (DAC) allows you to manage your Dremio catalog (Spaces, Folders, Views) using local files, enabling GitOps workflows.
4
+
5
+ ## Quick Start
6
+
7
+ ### 1. Initialize Configuration
8
+ Create a `dremio.yaml` file in your project root:
9
+
10
+ ```yaml
11
+ version: "1.0"
12
+ scope:
13
+ path: "dremio-catalog.finance" # The Dremio path to sync
14
+ type: "SPACE" # SPACE or ICEBERGCATALOG
15
+ ignore:
16
+ - "*.tmp"
17
+ ```
18
+
19
+ ### 2. Pull State
20
+ Capture the current state of your Dremio space into local files.
21
+
22
+ ```bash
23
+ dremio sync pull
24
+ ```
25
+
26
+ Returns a directory structure mirroring Dremio:
27
+ - `finance/`
28
+ - `monthly_report.sql`
29
+ - `monthly_report.yaml`
30
+
31
+ ### View Definition (`view.yaml`)
32
+
33
+ Define your virtual datasets with SQL, tags, wiki content, dependencies, and governance policies.
34
+
35
+ ```yaml
36
+ name: revenue_report
37
+ type: VIRTUAL_DATASET
38
+ # Full path in Dremio
39
+ path: ["dremio-catalog", "finance", "reports", "revenue_report"]
40
+ # SQL Definition
41
+ sql: |
42
+ SELECT region, sum(amount) as total
43
+ FROM "dremio-catalog".finance.stg_sales
44
+ GROUP BY region
45
+ # Dependencies
46
+ dependencies:
47
+ - "stg_sales"
48
+ # Tags & Wiki
49
+ tags: ["finance", "official"]
50
+ description: "docs/revenue_report.md"
51
+
52
+ # Governance: Access Control (RBAC)
53
+ access_control:
54
+ roles:
55
+ - name: "finance_managers"
56
+ privileges: ["SELECT"]
57
+ users:
58
+ - name: "auditor@example.com"
59
+ privileges: ["SELECT", "ALTER"]
60
+
61
+ # Governance: Row/Column Policies
62
+ # (Requires UDFs to be defined separately)
63
+ governance:
64
+ row_access_policy:
65
+ name: "protect_region_udf"
66
+ args: ["region"]
67
+ masking_policies:
68
+ - column: "total"
69
+ name: "mask_amount_udf"
70
+
71
+ # Reflections
72
+ reflections:
73
+ - name: "raw_sales_agg"
74
+ type: "RAW"
75
+ displayFields: ["region", "total"]
76
+ - name: "agg_sales_by_region"
77
+ type: "AGGREGATION"
78
+ dimensionFields: ["region"]
79
+ measureFields: ["total"]
80
+ distributionFields: ["region"]
81
+ partitionFields: ["region"]
82
+ ```
83
+
84
+ ### Workflow
85
+
86
+ 1. **Push**: `dremio sync push`
87
+ - recurses, sorts dependencies, applies SQL, updates Tags/Wiki.
88
+ - **Applies Grants**: Resolves Role/User names to IDs and enforces access control.
89
+ - **Applies Policies**: Executes SQL commands to attach Row Access and Masking policies.
90
+
91
+ 2. **Pull**: `dremio sync pull`
92
+ - Fetches state, rebuilds folders, views, wikis.
93
+ - **Important Limitation**: Governance policies (RBAC, Row Access, Masking) and **Reflections** are **NOT** automatically retrieved from Dremio during a pull.
94
+ - To manage them via DAC, you must manually define `access_control`, `governance`, and `reflections` blocks in your YAML files.
95
+ - The `pull` command will only generate the standard SQL and metadata.
96
+
97
+ **new_view.sql**:
98
+ ```sql
99
+ SELECT * FROM parent.table
100
+ ```
101
+
102
+ **new_view.yaml**:
103
+ ```yaml
104
+ type: VIEW
105
+ path: ["dremio-catalog", "finance", "new_view"]
106
+ sql_file: "new_view.sql"
107
+ context: []
108
+ ```
109
+
110
+ ### 4. Push Changes
111
+ Apply your local changes back to Dremio.
112
+
113
+ ```bash
114
+ dremio sync push
115
+ ```
116
+
117
+ ## Feature Guides
118
+
119
+ - **[Sources](dac_sources.md)**: Manage Dremio Sources (S3, Nessie, Relational).
120
+ - **[Tables](dac_tables.md)**: Manage Physical Datasets and Iceberg Tables.
121
+ - **[Validations](dac_validations.md)**: Define Data Quality Checks.
122
+ - **[Reflections](dac_reflections.md)**: Manage RAW and AGGREGATION reflections.
123
+ - **[Governance](dac_governance.md)**: Manage Access Control (RBAC) and Row/Column Policies.
124
+
125
+ ## Concepts
126
+
127
+ - **Scope**: Limits the sync to a specific subtree to support multi-team environments.
128
+ - **State File**: `.dremio_state.json` tracks the last known state to enable efficient updates and deletes. **Do not edit this file manually.**
@@ -0,0 +1,88 @@
1
+
2
+ # Governance in Dremio-as-Code
3
+
4
+ DAC allows you to manage Access Control (RBAC) and Fine-Grained Access Control (Row/Column Policies) as code.
5
+
6
+ ## 1. Access Control (RBAC)
7
+
8
+ Define who can access your dataset using the `access_control` block.
9
+
10
+ ### YAML Schema
11
+
12
+ ```yaml
13
+ name: "sensitive_data"
14
+ type: VIRTUAL_DATASET
15
+ ...
16
+
17
+ access_control:
18
+ roles:
19
+ - name: "PUBLIC"
20
+ privileges: ["SELECT"]
21
+ - name: "finance_analysts"
22
+ privileges: ["SELECT", "ALTER"]
23
+ users:
24
+ - name: "bob@example.com"
25
+ privileges: ["SELECT"]
26
+ ```
27
+
28
+ ### Privileges
29
+ Common privileges include:
30
+ - `SELECT`: Read data.
31
+ - `ALTER`: Modify definition.
32
+ - `MANAGE_GRANTS`: Change permissions.
33
+
34
+ ### Workflow
35
+ - **Push**: DAC resolves User/Role names to IDs and applies the grants.
36
+ - *Note*: This **replaces** existing grants on the dataset.
37
+
38
+ ## 2. Row Access Policies
39
+
40
+ Filter rows based on user context at query time.
41
+
42
+ ### YAML Schema
43
+
44
+ ```yaml
45
+ governance:
46
+ row_access_policy:
47
+ name: "dremio.security.region_policy"
48
+ args: ["region_id"]
49
+ ```
50
+
51
+ - **name**: Full path to the UDF (User Defined Function) that implements the logic.
52
+ - **args**: Columns from the dataset to pass as arguments to the UDF.
53
+
54
+ ### UDF Example
55
+ You must create the UDF in Dremio first (or via `create_sql` in DAC).
56
+ ```sql
57
+ CREATE FUNCTION dremio.security.region_policy (region_id VARCHAR)
58
+ RETURNS BOOLEAN
59
+ RETURN query_user() = 'admin' OR region_id = 'US'
60
+ ```
61
+
62
+ ## 3. Masking Policies (Column Level)
63
+
64
+ Mask sensitive column values.
65
+
66
+ ### YAML Schema
67
+
68
+ ```yaml
69
+ governance:
70
+ masking_policies:
71
+ - name: "dremio.security.mask_ssn"
72
+ column: "ssn"
73
+ args: ["ssn"]
74
+ - name: "dremio.security.mask_email"
75
+ column: "email"
76
+ args: ["email"]
77
+ ```
78
+
79
+ - **name**: Full path to the Masking UDF.
80
+ - **column**: The column to apply the mask to.
81
+ - **args**: Columns to pass to the UDF.
82
+
83
+ ### Workflow
84
+ - **Push**: DAC executes `ALTER VIEW ... ADD ROW ACCESS POLICY` or `MODIFY COLUMN ... SET MASKING POLICY`.
85
+ - **Limitations**:
86
+ - Only supported on Views (`VIRTUAL_DATASET`).
87
+ - Requires existing UDFs.
88
+ - Removing a policy from YAML does **not** automatically remove it from Dremio (requires manual `DROP` or `UNSET` logic, currently additive).
@@ -0,0 +1,63 @@
1
+
2
+ # Managing Reflections in Dremio-as-Code
3
+
4
+ Reflections are Dremio's query acceleration technology. DAC allows you to define and manage reflections alongside your dataset definitions.
5
+
6
+ ## YAML Schema
7
+
8
+ Add a `reflections` list to your Dataset YAML (`VIRTUAL_DATASET`, `ICEBERG_TABLE`, or `PHYSICAL_DATASET`).
9
+
10
+ ```yaml
11
+ name: "orders"
12
+ type: VIRTUAL_DATASET
13
+ path: ["dremio", "business", "orders"]
14
+ sql: "SELECT * FROM source.orders"
15
+
16
+ reflections:
17
+ # Raw Reflection
18
+ - name: "raw_orders"
19
+ type: "RAW"
20
+ displayFields: ["order_id", "customer_id", "amount", "order_date"]
21
+ partitionFields: ["order_date"]
22
+ distributionFields: ["customer_id"]
23
+ enabled: true
24
+
25
+ # Aggregation Reflection
26
+ - name: "agg_orders_by_customer"
27
+ type: "AGGREGATION"
28
+ dimensionFields: ["customer_id", "order_year"]
29
+ measureFields: ["amount"]
30
+ partitionFields: ["order_year"]
31
+ distributionFields: ["customer_id"]
32
+ enabled: true
33
+ ```
34
+
35
+ ## Reflection Types
36
+
37
+ ### RAW
38
+ Accelerates detailed queries.
39
+ - `displayFields`: List of columns to include.
40
+ - `partitionFields`: Columns to partition the reflection by.
41
+ - `distributionFields`: Columns to distribute data across nodes.
42
+
43
+ ### AGGREGATION
44
+ Accelerates aggregate queries (GROUP BY).
45
+ - `dimensionFields`: Columns used in GROUP BY.
46
+ - `measureFields`: Columns used in aggregate functions (SUM, COUNT, etc.).
47
+ - `partitionFields`: Partitioning configuration.
48
+ - `distributionFields`: Distribution configuration.
49
+
50
+ ## Workflow
51
+
52
+ 1. **Push**:
53
+ - DAC checks for existing reflections on the dataset.
54
+ - Matches by **Name**.
55
+ - **Update**: If found, updates configuration (enabled, fields).
56
+ - **Create**: If missing, creates the reflection.
57
+ 2. **Pull**:
58
+ - *Limitation*: Currently, pulling reflections from existing datasets is not fully supported. You must define them manually in your YAML.
59
+
60
+ ## Best Practices
61
+ - Use descriptive names (e.g., `agg_by_region`).
62
+ - Keep definitions in the same YAML file as the View/Table.
63
+ - Use `enabled: false` to disable a reflection without deleting the definition.
@@ -0,0 +1,80 @@
1
+
2
+ # Managing Sources with Dremio-as-Code
3
+
4
+ DAC allows you to define and manage Dremio Sources (e.g., S3, Postgres, Nessie, Arctic) using YAML configuration files.
5
+
6
+ ## Overview
7
+
8
+ - **Secure**: Use Environment Variable substitution (`${ENV_VAR}`) to keep credentials out of code.
9
+ - **Declarative**: Define the source configuration, and `dremio sync push` handles creation or updates.
10
+ - **Top-Level**: Sources are usually defined in the root of your DAC directory or a dedicated `sources/` folder.
11
+
12
+ ## YAML Schema
13
+
14
+ ```yaml
15
+ name: "my-source-name"
16
+ type: SOURCE
17
+ path: ["my-source-name"] # Must match name
18
+ source_type: "S3" # Dremio Source Type Code (e.g. S3, POSTGRES, NESSIE, ARCTIC, ADL)
19
+ config:
20
+ # Source specific configuration
21
+ accessKey: "${AWS_ACCESS_KEY}"
22
+ secretKey: "${AWS_SECRET_KEY}"
23
+ rootPath: "/"
24
+ metadata_policy:
25
+ authTtlMs: 3600000
26
+ ...
27
+ ```
28
+
29
+ ## Examples
30
+
31
+ ### 1. Amazon S3
32
+
33
+ ```yaml
34
+ name: "datalake-s3"
35
+ type: SOURCE
36
+ source_type: "S3"
37
+ config:
38
+ accessKey: "${AWS_ACCESS_KEY}"
39
+ secretKey: "${AWS_SECRET_KEY}"
40
+ secure: true
41
+ rootPath: "/my-bucket/data"
42
+ ```
43
+
44
+ ### 2. Postgres
45
+
46
+ ```yaml
47
+ name: "app-db"
48
+ type: SOURCE
49
+ source_type: "POSTGRES"
50
+ config:
51
+ hostname: "db.production.internal"
52
+ port: 5432
53
+ username: "dremio_user"
54
+ password: "${PG_PASSWORD}"
55
+ databaseName: "myapp"
56
+ ```
57
+
58
+ ### 3. Nessie / Arctic
59
+
60
+ ```yaml
61
+ name: "arctic-catalog"
62
+ type: SOURCE
63
+ source_type: "NESSIE"
64
+ config:
65
+ endpoint: "https://nessie.dremio.cloud/v1/projects/${PROJECT_ID}"
66
+ authType: "BEARER"
67
+ token: "${NESSIE_TOKEN}"
68
+ ```
69
+
70
+ ## Environment Variables
71
+
72
+ Prior to running `dremio sync push`, ensure the referenced environment variables are set in your shell or `.env` file.
73
+
74
+ ```bash
75
+ export AWS_ACCESS_KEY="AKI..."
76
+ export AWS_SECRET_KEY="secret..."
77
+ dremio sync push
78
+ ```
79
+
80
+ If a variable is missing, the CLI will warn you and keep the literal string (which leads to auth failure), protecting you from accidental commits of unexpanded secrets.
@@ -0,0 +1,78 @@
1
+
2
+ # Managing Tables in Dremio-as-Code
3
+
4
+ DAC supports managing Physical Iceberg Tables and standard Physical Datasets.
5
+
6
+ ## 1. Iceberg Tables (`type: ICEBERG_TABLE`)
7
+
8
+ You can use DAC to manage the lifecycle of Dremio-native Iceberg tables (e.g., in Arctic, Nessie, or S3). This allows for "Lightweight ETL" where every `push` triggers an SQL update script.
9
+
10
+ ### YAML Schema
11
+
12
+ ```yaml
13
+ name: "app_events"
14
+ type: ICEBERG_TABLE
15
+ path: ["dremio", "etl", "app_events"]
16
+
17
+ # Run ONE-TIME if table doesn't exist
18
+ create_sql: |
19
+ CREATE TABLE dremio.etl.app_events (
20
+ id VARCHAR,
21
+ event_time TIMESTAMP,
22
+ payload VARCHAR
23
+ ) PARTITION BY (event_time)
24
+
25
+ # Run EVERY PUSH (if table exists)
26
+ update_sql: |
27
+ MERGE INTO dremio.etl.app_events t
28
+ USING source.raw_events s
29
+ ON t.id = s.id
30
+ WHEN MATCHED THEN UPDATE SET t.payload = s.payload
31
+ WHEN NOT MATCHED THEN INSERT VALUES (s.id, s.event_time, s.payload)
32
+
33
+ # Standard features
34
+ tags: ["events", "etl"]
35
+ governance: ...
36
+ reflections: ...
37
+ ```
38
+
39
+ ### Workflow
40
+ 1. **First Push**: CLI detects missing table. executes `create_sql`.
41
+ 2. **Subsequent Pushes**: CLI detects existing table. Executes `update_sql`.
42
+
43
+ ### Pulling Iceberg Tables
44
+ `dremio sync pull` will generate the YAML for existing Iceberg tables.
45
+ *Limitation*: The CLI cannot reconstruct the `create_sql` or `update_sql` logic. The fields will be generated as placeholders or comments.
46
+
47
+ ## 2. Physical Datasets (`type: PHYSICAL_DATASET`)
48
+
49
+ For external files (S3 Parquet/CSV) or RDBMS tables that are read-only to Dremio, use `PHYSICAL_DATASET`. DAC manages their **metadata** (Governance, Reflections, Wiki), not their data lifecycle.
50
+
51
+ ### YAML Schema
52
+
53
+ ```yaml
54
+ name: "raw_customers"
55
+ type: PHYSICAL_DATASET
56
+ path: ["s3-source", "bucket", "customers.parquet"]
57
+
58
+ # Metadata
59
+ tags: ["raw", "source"]
60
+ description: "docs/raw_customers.md"
61
+
62
+ # Format (Optional - for auto-promotion)
63
+ format:
64
+ type: "Parquet"
65
+
66
+ # Governance
67
+ access_control:
68
+ roles:
69
+ - name: "analysts"
70
+ privileges: ["SELECT"]
71
+ reflections:
72
+ - name: "raw_ref"
73
+ type: "RAW"
74
+ ```
75
+
76
+ ### Workflow
77
+ - **Push**: Ensures the dataset is promoted/exists. Applies tags, wiki, grants, and reflections.
78
+ - **Pull**: Generates YAML for existing promoted datasets in the scope.
@@ -0,0 +1,54 @@
1
+
2
+ # Data Validations in Dremio-as-Code
3
+
4
+ Ensure data quality and integrity by defining SQL assertions that run after every `sync push`.
5
+
6
+ ## Overview
7
+
8
+ Validations are SQL queries paired with a condition. The CLI executes the query, fetches the scalar result (first column of first row), and compares it against your condition.
9
+
10
+ ## YAML Schema
11
+
12
+ Add a `validations` block to any dataset YAML (`VIRTUAL_DATASET` or `ICEBERG_TABLE`).
13
+
14
+ ```yaml
15
+ name: "mart_revenue"
16
+ type: VIRTUAL_DATASET
17
+ path: ["dremio", "finance", "mart_revenue"]
18
+ sql: "SELECT ..."
19
+
20
+ validations:
21
+ # Check 1: Non-zero rows
22
+ - name: "row_count"
23
+ sql: "SELECT count(*) FROM dremio.finance.mart_revenue"
24
+ condition: "gt 0"
25
+
26
+ # Check 2: No null regions
27
+ - name: "no_null_regions"
28
+ sql: "SELECT count(*) FROM dremio.finance.mart_revenue WHERE region IS NULL"
29
+ condition: "eq 0"
30
+
31
+ # Check 3: Total is positive
32
+ - name: "positive_revenue"
33
+ sql: "SELECT min(total_revenue) FROM dremio.finance.mart_revenue"
34
+ condition: "gte 0"
35
+ ```
36
+
37
+ ## Supported Conditions
38
+
39
+ - `eq {val}`: Equal to value
40
+ - `neq {val}`: Not equal to value
41
+ - `gt {val}`: Greater than value
42
+ - `lt {val}`: Less than value
43
+ - `gte {val}`: Greater than or equal to value
44
+ - `lte {val}`: Less than or equal to value
45
+
46
+ ## Workflow
47
+
48
+ 1. Sync (Create/Update View/Table).
49
+ 2. Wait for completion.
50
+ 3. **Run Validations**:
51
+ - Execute Check SQL.
52
+ - Fetch Result.
53
+ - Evaluate Condition.
54
+ - Log `[PASS]` or `[FAIL]`.
@@ -0,0 +1,139 @@
1
+ # Profile Management Guide
2
+
3
+ This guide covers how to create and manage Dremio CLI profiles using both YAML configuration files and environment variables.
4
+
5
+ ## Overview
6
+
7
+ Profiles store connection information for Dremio instances. The CLI supports two methods:
8
+
9
+ 1. **YAML Configuration** - Stored in `~/.dremio/profiles.yaml` (RECOMMENDED for local use)
10
+ 2. **Environment Variables** - Loaded from `.env` file or shell environment (RECOMMENDED for CI/CD)
11
+
12
+ Environment variables take precedence over YAML profiles.
13
+
14
+ ---
15
+
16
+ ## 🚀 Profile Values Guide
17
+
18
+ ### 1. Base URL
19
+ The URL to your Dremio instance's API.
20
+
21
+ | Platform | Format | Example | Note |
22
+ |----------|--------|---------|------|
23
+ | **Dremio Cloud** | `https://api.dremio.cloud/v0` | `https://api.dremio.cloud/v0` | Used for US control plane |
24
+ | **Dremio Cloud (EU)** | `https://api.dremio.eu/v0` | `https://api.dremio.eu/v0` | Used for EU control plane |
25
+ | **Dremio Software** | `http(s)://<host>:<port>` | `https://dremio.company.com` | **Smart URL**: The CLI automatically appends `/api/v3` if you omit it. |
26
+ | **Local Software** | `http://localhost:9047` | `http://localhost:9047` | Defaults for local Docker/install |
27
+
28
+ > **Note:** For Dremio Software, you can provide `https://dremio.company.com` OR `https://dremio.company.com/api/v3`. The CLI handles both correctly.
29
+
30
+ ### 2. Authentication
31
+ How you log in to Dremio.
32
+
33
+ | Type | Platform | Description |
34
+ |------|----------|-------------|
35
+ | **PAT** (Token) | Cloud & Software | **Recommended**. Personal Access Token generated in User Settings. |
36
+ | **Services Account** | Cloud Only | Treats Client/Secret as a PAT for automation. |
37
+ | **Username/Password** | Software Only | Traditional login. **Less secure** than PAT. |
38
+
39
+ ### 3. Project ID (Cloud Only)
40
+ Can be found in the URL of your Dremio Cloud project.
41
+ - URL: `https://app.dremio.cloud/projectId/12345-abcde.../home`
42
+ - Project ID: `12345-abcde...`
43
+
44
+ ---
45
+
46
+ ## YAML Configuration
47
+
48
+ **Location**: `~/.dremio/profiles.yaml`
49
+
50
+ ### Examples
51
+
52
+ **Dremio Cloud (US)**
53
+ ```yaml
54
+ profiles:
55
+ cloud-prod:
56
+ type: cloud
57
+ base_url: https://api.dremio.cloud/v0
58
+ project_id: 788baab4-3c3b-42da-9f1d-5cc6dc03147d
59
+ auth:
60
+ type: pat
61
+ token: your-personal-access-token
62
+ ```
63
+
64
+ **Dremio Software (Corporate)**
65
+ ```yaml
66
+ profiles:
67
+ corp-dremio:
68
+ type: software
69
+ base_url: https://dremio.corp.com # CLI will add /api/v3 automatically
70
+ auth:
71
+ type: pat
72
+ token: your-personal-access-token
73
+ ```
74
+
75
+ **Dremio Software (Local/Docker)**
76
+ ```yaml
77
+ profiles:
78
+ local:
79
+ type: software
80
+ base_url: http://localhost:9047 # Default port
81
+ auth:
82
+ type: username_password
83
+ username: admin
84
+ password: password123
85
+ ```
86
+
87
+ ### CLI Commands
88
+
89
+ ```bash
90
+ # Interactve Wizard (Best for beginners)
91
+ dremio init
92
+
93
+ # Create manually
94
+ dremio profile create --name prod --type cloud ...
95
+ ```
96
+
97
+ ---
98
+
99
+ ## Environment Variable Configuration
100
+
101
+ Ideal for CI/CD pipelines or Docker containers.
102
+
103
+ ### Pattern
104
+ `DREMIO_{PROFILE_NAME}_KEY=VALUE`
105
+
106
+ ### Example `.env` File
107
+ ```bash
108
+ # Cloud Profile (Name: 'CLOUD')
109
+ DREMIO_CLOUD_TYPE=cloud
110
+ DREMIO_CLOUD_BASE_URL=https://api.dremio.cloud/v0
111
+ DREMIO_CLOUD_PROJECTID=788baab4-3c3b-42da-9f1d-5cc6dc03147d
112
+ DREMIO_CLOUD_TOKEN=s3JcLOqFTR...
113
+
114
+ # Software Profile (Name: 'PROD')
115
+ DREMIO_PROD_TYPE=software
116
+ DREMIO_PROD_BASE_URL=https://dremio.corp.com
117
+ DREMIO_PROD_TOKEN=Q/ToosxORA...
118
+ ```
119
+
120
+ ### Usage
121
+ ```bash
122
+ # Authenticates using DREMIO_PROD_* variables
123
+ dremio --profile prod catalog list
124
+ ```
125
+
126
+ ---
127
+
128
+ ## Profile Management
129
+
130
+ ```bash
131
+ # List all profiles
132
+ dremio profile list
133
+
134
+ # Show active profile details
135
+ dremio profile current
136
+
137
+ # Set default profile (so you don't need --profile flag)
138
+ dremio profile set-default cloud-prod
139
+ ```