dremio-cli 1.7.0__tar.gz → 1.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. {dremio_cli-1.7.0/dremio_cli.egg-info → dremio_cli-1.8.0}/PKG-INFO +10 -1
  2. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/README.md +9 -0
  3. dremio_cli-1.8.0/docs/dac.md +128 -0
  4. dremio_cli-1.8.0/docs/dac_governance.md +88 -0
  5. dremio_cli-1.8.0/docs/dac_reflections.md +63 -0
  6. dremio_cli-1.8.0/docs/dac_sources.md +80 -0
  7. dremio_cli-1.8.0/docs/dac_tables.md +78 -0
  8. dremio_cli-1.8.0/docs/dac_validations.md +54 -0
  9. dremio_cli-1.8.0/dremio_cli/dac/graph.py +69 -0
  10. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/dac/state.py +47 -12
  11. dremio_cli-1.8.0/dremio_cli/dac/sync.py +863 -0
  12. {dremio_cli-1.7.0 → dremio_cli-1.8.0/dremio_cli.egg-info}/PKG-INFO +10 -1
  13. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli.egg-info/SOURCES.txt +6 -0
  14. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/pyproject.toml +1 -1
  15. dremio_cli-1.7.0/docs/dac.md +0 -60
  16. dremio_cli-1.7.0/dremio_cli/dac/sync.py +0 -344
  17. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/LICENSE +0 -0
  18. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/MANIFEST.in +0 -0
  19. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/completions/dremio-completion.bash +0 -0
  20. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/completions/dremio-completion.zsh +0 -0
  21. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/README.md +0 -0
  22. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/advanced-features.md +0 -0
  23. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/catalog.md +0 -0
  24. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/completion.md +0 -0
  25. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/favorites.md +0 -0
  26. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/grants.md +0 -0
  27. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/history.md +0 -0
  28. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/init.md +0 -0
  29. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/installation.md +0 -0
  30. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/jobs.md +0 -0
  31. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/lineage.md +0 -0
  32. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/monitor.md +0 -0
  33. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/profiles.md +0 -0
  34. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/quickstart.md +0 -0
  35. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/reflections.md +0 -0
  36. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/repl.md +0 -0
  37. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/roles.md +0 -0
  38. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/scripts.md +0 -0
  39. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/sources.md +0 -0
  40. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/spaces-folders.md +0 -0
  41. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/sql.md +0 -0
  42. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/tables.md +0 -0
  43. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/tags-wiki.md +0 -0
  44. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/tui.md +0 -0
  45. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/users.md +0 -0
  46. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/docs/views.md +0 -0
  47. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/__init__.py +0 -0
  48. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/__main__.py +0 -0
  49. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/cli.py +0 -0
  50. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/client/__init__.py +0 -0
  51. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/client/auth.py +0 -0
  52. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/client/base.py +0 -0
  53. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/client/cloud.py +0 -0
  54. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/client/factory.py +0 -0
  55. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/client/software.py +0 -0
  56. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/__init__.py +0 -0
  57. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/catalog.py +0 -0
  58. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/completion.py +0 -0
  59. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/favorite.py +0 -0
  60. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/folder.py +0 -0
  61. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/grant.py +0 -0
  62. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/history.py +0 -0
  63. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/init.py +0 -0
  64. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/job.py +0 -0
  65. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/lineage.py +0 -0
  66. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/monitor.py +0 -0
  67. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/profile.py +0 -0
  68. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/reflection.py +0 -0
  69. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/repl.py +0 -0
  70. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/role.py +0 -0
  71. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/script.py +0 -0
  72. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/source.py +0 -0
  73. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/space.py +0 -0
  74. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/sql.py +0 -0
  75. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/sync.py +0 -0
  76. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/table.py +0 -0
  77. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/tag.py +0 -0
  78. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/udf.py +0 -0
  79. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/ui.py +0 -0
  80. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/user.py +0 -0
  81. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/view.py +0 -0
  82. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/commands/wiki.py +0 -0
  83. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/config.py +0 -0
  84. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/dac/__init__.py +0 -0
  85. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/dac/config.py +0 -0
  86. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/dac/diff.py +0 -0
  87. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/env.py +0 -0
  88. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/formatters/__init__.py +0 -0
  89. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/formatters/json.py +0 -0
  90. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/formatters/table.py +0 -0
  91. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/formatters/yaml.py +0 -0
  92. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/history.py +0 -0
  93. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/tui/catalog/__init__.py +0 -0
  94. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/tui/catalog/app.py +0 -0
  95. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/tui/catalog/screens.py +0 -0
  96. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/tui/catalog/widgets.py +0 -0
  97. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/utils/__init__.py +0 -0
  98. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/utils/console.py +0 -0
  99. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/utils/exceptions.py +0 -0
  100. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/utils/helpers.py +0 -0
  101. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli/utils/validators.py +0 -0
  102. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli.egg-info/dependency_links.txt +0 -0
  103. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli.egg-info/entry_points.txt +0 -0
  104. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli.egg-info/requires.txt +0 -0
  105. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/dremio_cli.egg-info/top_level.txt +0 -0
  106. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/setup.cfg +0 -0
  107. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/setup.py +0 -0
  108. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/__init__.py +0 -0
  109. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/conftest.py +0 -0
  110. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_config.py +0 -0
  111. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_grant_commands.py +0 -0
  112. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_history_favorite.py +0 -0
  113. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_integration.py +0 -0
  114. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_job_commands.py +0 -0
  115. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_multi_sql_execution.py +0 -0
  116. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_reflection_commands.py +0 -0
  117. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_role_commands.py +0 -0
  118. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_script_commands.py +0 -0
  119. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_source_commands.py +0 -0
  120. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_space_folder_commands.py +0 -0
  121. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_sql_commands.py +0 -0
  122. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_table_commands.py +0 -0
  123. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_tag_wiki_commands.py +0 -0
  124. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_user_commands.py +0 -0
  125. {dremio_cli-1.7.0 → dremio_cli-1.8.0}/tests/test_view_commands.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dremio-cli
3
- Version: 1.7.0
3
+ Version: 1.8.0
4
4
  Summary: A comprehensive command-line interface for Dremio Cloud and Dremio Software
5
5
  Home-page: https://github.com/your-org/dremio-cli
6
6
  Author: Your Name
@@ -119,6 +119,15 @@ dremio view create --path "Analytics.summary" \
119
119
  - **[Reflections](docs/reflections.md)** - Manage reflections
120
120
  - **[Scripts](docs/scripts.md)** - Manage scripts (Cloud only)
121
121
 
122
+ ### Dremio-as-Code (GitOps)
123
+
124
+ - **[Overview](docs/dac.md)** - Getting Started with DAC
125
+ - **[Sources](docs/dac_sources.md)** - Manage Sources
126
+ - **[Tables](docs/dac_tables.md)** - Manage Physical Tables
127
+ - **[Validations](docs/dac_validations.md)** - Data Quality Checks
128
+ - **[Reflections](docs/dac_reflections.md)** - Manage Reflections
129
+ - **[Governance](docs/dac_governance.md)** - Managing Access Control
130
+
122
131
  ### Productivity
123
132
 
124
133
  - **[Favorites](docs/favorites.md)** - Save frequently used queries
@@ -66,6 +66,15 @@ dremio view create --path "Analytics.summary" \
66
66
  - **[Reflections](docs/reflections.md)** - Manage reflections
67
67
  - **[Scripts](docs/scripts.md)** - Manage scripts (Cloud only)
68
68
 
69
+ ### Dremio-as-Code (GitOps)
70
+
71
+ - **[Overview](docs/dac.md)** - Getting Started with DAC
72
+ - **[Sources](docs/dac_sources.md)** - Manage Sources
73
+ - **[Tables](docs/dac_tables.md)** - Manage Physical Tables
74
+ - **[Validations](docs/dac_validations.md)** - Data Quality Checks
75
+ - **[Reflections](docs/dac_reflections.md)** - Manage Reflections
76
+ - **[Governance](docs/dac_governance.md)** - Managing Access Control
77
+
69
78
  ### Productivity
70
79
 
71
80
  - **[Favorites](docs/favorites.md)** - Save frequently used queries
@@ -0,0 +1,128 @@
1
+ # Dremio-as-Code (GitOps)
2
+
3
+ Dremio-as-Code (DAC) allows you to manage your Dremio catalog (Spaces, Folders, Views) using local files, enabling GitOps workflows.
4
+
5
+ ## Quick Start
6
+
7
+ ### 1. Initialize Configuration
8
+ Create a `dremio.yaml` file in your project root:
9
+
10
+ ```yaml
11
+ version: "1.0"
12
+ scope:
13
+ path: "dremio-catalog.finance" # The Dremio path to sync
14
+ type: "SPACE" # SPACE or ICEBERGCATALOG
15
+ ignore:
16
+ - "*.tmp"
17
+ ```
18
+
19
+ ### 2. Pull State
20
+ Capture the current state of your Dremio space into local files.
21
+
22
+ ```bash
23
+ dremio sync pull
24
+ ```
25
+
26
+ Returns a directory structure mirroring Dremio:
27
+ - `finance/`
28
+ - `monthly_report.sql`
29
+ - `monthly_report.yaml`
30
+
31
+ ### View Definition (`view.yaml`)
32
+
33
+ Define your virtual datasets with SQL, tags, wiki content, dependencies, and governance policies.
34
+
35
+ ```yaml
36
+ name: revenue_report
37
+ type: VIRTUAL_DATASET
38
+ # Full path in Dremio
39
+ path: ["dremio-catalog", "finance", "reports", "revenue_report"]
40
+ # SQL Definition
41
+ sql: |
42
+ SELECT region, sum(amount) as total
43
+ FROM "dremio-catalog".finance.stg_sales
44
+ GROUP BY region
45
+ # Dependencies
46
+ dependencies:
47
+ - "stg_sales"
48
+ # Tags & Wiki
49
+ tags: ["finance", "official"]
50
+ description: "docs/revenue_report.md"
51
+
52
+ # Governance: Access Control (RBAC)
53
+ access_control:
54
+ roles:
55
+ - name: "finance_managers"
56
+ privileges: ["SELECT"]
57
+ users:
58
+ - name: "auditor@example.com"
59
+ privileges: ["SELECT", "ALTER"]
60
+
61
+ # Governance: Row/Column Policies
62
+ # (Requires UDFs to be defined separately)
63
+ governance:
64
+ row_access_policy:
65
+ name: "protect_region_udf"
66
+ args: ["region"]
67
+ masking_policies:
68
+ - column: "total"
69
+ name: "mask_amount_udf"
70
+
71
+ # Reflections
72
+ reflections:
73
+ - name: "raw_sales_agg"
74
+ type: "RAW"
75
+ displayFields: ["region", "total"]
76
+ - name: "agg_sales_by_region"
77
+ type: "AGGREGATION"
78
+ dimensionFields: ["region"]
79
+ measureFields: ["total"]
80
+ distributionFields: ["region"]
81
+ partitionFields: ["region"]
82
+ ```
83
+
84
+ ### Workflow
85
+
86
+ 1. **Push**: `dremio sync push`
87
+ - recurses, sorts dependencies, applies SQL, updates Tags/Wiki.
88
+ - **Applies Grants**: Resolves Role/User names to IDs and enforces access control.
89
+ - **Applies Policies**: Executes SQL commands to attach Row Access and Masking policies.
90
+
91
+ 2. **Pull**: `dremio sync pull`
92
+ - Fetches state, rebuilds folders, views, wikis.
93
+ - **Important Limitation**: Governance policies (RBAC, Row Access, Masking) and **Reflections** are **NOT** automatically retrieved from Dremio during a pull.
94
+ - To manage them via DAC, you must manually define `access_control`, `governance`, and `reflections` blocks in your YAML files.
95
+ - The `pull` command will only generate the standard SQL and metadata.
96
+
97
+ **new_view.sql**:
98
+ ```sql
99
+ SELECT * FROM parent.table
100
+ ```
101
+
102
+ **new_view.yaml**:
103
+ ```yaml
104
+ type: VIEW
105
+ path: ["dremio-catalog", "finance", "new_view"]
106
+ sql_file: "new_view.sql"
107
+ context: []
108
+ ```
109
+
110
+ ### 4. Push Changes
111
+ Apply your local changes back to Dremio.
112
+
113
+ ```bash
114
+ dremio sync push
115
+ ```
116
+
117
+ ## Feature Guides
118
+
119
+ - **[Sources](dac_sources.md)**: Manage Dremio Sources (S3, Nessie, Relational).
120
+ - **[Tables](dac_tables.md)**: Manage Physical Datasets and Iceberg Tables.
121
+ - **[Validations](dac_validations.md)**: Define Data Quality Checks.
122
+ - **[Reflections](dac_reflections.md)**: Manage RAW and AGGREGATION reflections.
123
+ - **[Governance](dac_governance.md)**: Manage Access Control (RBAC) and Row/Column Policies.
124
+
125
+ ## Concepts
126
+
127
+ - **Scope**: Limits the sync to a specific subtree to support multi-team environments.
128
+ - **State File**: `.dremio_state.json` tracks the last known state to enable efficient updates and deletes. **Do not edit this file manually.**
@@ -0,0 +1,88 @@
1
+
2
+ # Governance in Dremio-as-Code
3
+
4
+ DAC allows you to manage Access Control (RBAC) and Fine-Grained Access Control (Row/Column Policies) as code.
5
+
6
+ ## 1. Access Control (RBAC)
7
+
8
+ Define who can access your dataset using the `access_control` block.
9
+
10
+ ### YAML Schema
11
+
12
+ ```yaml
13
+ name: "sensitive_data"
14
+ type: VIRTUAL_DATASET
15
+ ...
16
+
17
+ access_control:
18
+ roles:
19
+ - name: "PUBLIC"
20
+ privileges: ["SELECT"]
21
+ - name: "finance_analysts"
22
+ privileges: ["SELECT", "ALTER"]
23
+ users:
24
+ - name: "bob@example.com"
25
+ privileges: ["SELECT"]
26
+ ```
27
+
28
+ ### Privileges
29
+ Common privileges include:
30
+ - `SELECT`: Read data.
31
+ - `ALTER`: Modify definition.
32
+ - `MANAGE_GRANTS`: Change permissions.
33
+
34
+ ### Workflow
35
+ - **Push**: DAC resolves User/Role names to IDs and applies the grants.
36
+ - *Note*: This **replaces** existing grants on the dataset.
37
+
38
+ ## 2. Row Access Policies
39
+
40
+ Filter rows based on user context at query time.
41
+
42
+ ### YAML Schema
43
+
44
+ ```yaml
45
+ governance:
46
+ row_access_policy:
47
+ name: "dremio.security.region_policy"
48
+ args: ["region_id"]
49
+ ```
50
+
51
+ - **name**: Full path to the UDF (User Defined Function) that implements the logic.
52
+ - **args**: Columns from the dataset to pass as arguments to the UDF.
53
+
54
+ ### UDF Example
55
+ You must create the UDF in Dremio first (or via `create_sql` in DAC).
56
+ ```sql
57
+ CREATE FUNCTION dremio.security.region_policy (region_id VARCHAR)
58
+ RETURNS BOOLEAN
59
+ RETURN query_user() = 'admin' OR region_id = 'US'
60
+ ```
61
+
62
+ ## 3. Masking Policies (Column Level)
63
+
64
+ Mask sensitive column values.
65
+
66
+ ### YAML Schema
67
+
68
+ ```yaml
69
+ governance:
70
+ masking_policies:
71
+ - name: "dremio.security.mask_ssn"
72
+ column: "ssn"
73
+ args: ["ssn"]
74
+ - name: "dremio.security.mask_email"
75
+ column: "email"
76
+ args: ["email"]
77
+ ```
78
+
79
+ - **name**: Full path to the Masking UDF.
80
+ - **column**: The column to apply the mask to.
81
+ - **args**: Columns to pass to the UDF.
82
+
83
+ ### Workflow
84
+ - **Push**: DAC executes `ALTER VIEW ... ADD ROW ACCESS POLICY` or `MODIFY COLUMN ... SET MASKING POLICY`.
85
+ - **Limitations**:
86
+ - Only supported on Views (`VIRTUAL_DATASET`).
87
+ - Requires existing UDFs.
88
+ - Removing a policy from YAML does **not** automatically remove it from Dremio (requires manual `DROP` or `UNSET` logic, currently additive).
@@ -0,0 +1,63 @@
1
+
2
+ # Managing Reflections in Dremio-as-Code
3
+
4
+ Reflections are Dremio's query acceleration technology. DAC allows you to define and manage reflections alongside your dataset definitions.
5
+
6
+ ## YAML Schema
7
+
8
+ Add a `reflections` list to your Dataset YAML (`VIRTUAL_DATASET`, `ICEBERG_TABLE`, or `PHYSICAL_DATASET`).
9
+
10
+ ```yaml
11
+ name: "orders"
12
+ type: VIRTUAL_DATASET
13
+ path: ["dremio", "business", "orders"]
14
+ sql: "SELECT * FROM source.orders"
15
+
16
+ reflections:
17
+ # Raw Reflection
18
+ - name: "raw_orders"
19
+ type: "RAW"
20
+ displayFields: ["order_id", "customer_id", "amount", "order_date"]
21
+ partitionFields: ["order_date"]
22
+ distributionFields: ["customer_id"]
23
+ enabled: true
24
+
25
+ # Aggregation Reflection
26
+ - name: "agg_orders_by_customer"
27
+ type: "AGGREGATION"
28
+ dimensionFields: ["customer_id", "order_year"]
29
+ measureFields: ["amount"]
30
+ partitionFields: ["order_year"]
31
+ distributionFields: ["customer_id"]
32
+ enabled: true
33
+ ```
34
+
35
+ ## Reflection Types
36
+
37
+ ### RAW
38
+ Accelerates detailed queries.
39
+ - `displayFields`: List of columns to include.
40
+ - `partitionFields`: Columns to partition the reflection by.
41
+ - `distributionFields`: Columns to distribute data across nodes.
42
+
43
+ ### AGGREGATION
44
+ Accelerates aggregate queries (GROUP BY).
45
+ - `dimensionFields`: Columns used in GROUP BY.
46
+ - `measureFields`: Columns used in aggregate functions (SUM, COUNT, etc.).
47
+ - `partitionFields`: Partitioning configuration.
48
+ - `distributionFields`: Distribution configuration.
49
+
50
+ ## Workflow
51
+
52
+ 1. **Push**:
53
+ - DAC checks for existing reflections on the dataset.
54
+ - Matches by **Name**.
55
+ - **Update**: If found, updates configuration (enabled, fields).
56
+ - **Create**: If missing, creates the reflection.
57
+ 2. **Pull**:
58
+ - *Limitation*: Currently, pulling reflections from existing datasets is not fully supported. You must define them manually in your YAML.
59
+
60
+ ## Best Practices
61
+ - Use descriptive names (e.g., `agg_by_region`).
62
+ - Keep definitions in the same YAML file as the View/Table.
63
+ - Use `enabled: false` to disable a reflection without deleting the definition.
@@ -0,0 +1,80 @@
1
+
2
+ # Managing Sources with Dremio-as-Code
3
+
4
+ DAC allows you to define and manage Dremio Sources (e.g., S3, Postgres, Nessie, Arctic) using YAML configuration files.
5
+
6
+ ## Overview
7
+
8
+ - **Secure**: Use Environment Variable substitution (`${ENV_VAR}`) to keep credentials out of code.
9
+ - **Declarative**: Define the source configuration, and `dremio sync push` handles creation or updates.
10
+ - **Top-Level**: Sources are usually defined in the root of your DAC directory or a dedicated `sources/` folder.
11
+
12
+ ## YAML Schema
13
+
14
+ ```yaml
15
+ name: "my-source-name"
16
+ type: SOURCE
17
+ path: ["my-source-name"] # Must match name
18
+ source_type: "S3" # Dremio Source Type Code (e.g. S3, POSTGRES, NESSIE, ARCTIC, ADL)
19
+ config:
20
+ # Source specific configuration
21
+ accessKey: "${AWS_ACCESS_KEY}"
22
+ secretKey: "${AWS_SECRET_KEY}"
23
+ rootPath: "/"
24
+ metadata_policy:
25
+ authTtlMs: 3600000
26
+ ...
27
+ ```
28
+
29
+ ## Examples
30
+
31
+ ### 1. Amazon S3
32
+
33
+ ```yaml
34
+ name: "datalake-s3"
35
+ type: SOURCE
36
+ source_type: "S3"
37
+ config:
38
+ accessKey: "${AWS_ACCESS_KEY}"
39
+ secretKey: "${AWS_SECRET_KEY}"
40
+ secure: true
41
+ rootPath: "/my-bucket/data"
42
+ ```
43
+
44
+ ### 2. Postgres
45
+
46
+ ```yaml
47
+ name: "app-db"
48
+ type: SOURCE
49
+ source_type: "POSTGRES"
50
+ config:
51
+ hostname: "db.production.internal"
52
+ port: 5432
53
+ username: "dremio_user"
54
+ password: "${PG_PASSWORD}"
55
+ databaseName: "myapp"
56
+ ```
57
+
58
+ ### 3. Nessie / Arctic
59
+
60
+ ```yaml
61
+ name: "arctic-catalog"
62
+ type: SOURCE
63
+ source_type: "NESSIE"
64
+ config:
65
+ endpoint: "https://nessie.dremio.cloud/v1/projects/${PROJECT_ID}"
66
+ authType: "BEARER"
67
+ token: "${NESSIE_TOKEN}"
68
+ ```
69
+
70
+ ## Environment Variables
71
+
72
+ Prior to running `dremio sync push`, ensure the referenced environment variables are set in your shell or `.env` file.
73
+
74
+ ```bash
75
+ export AWS_ACCESS_KEY="AKI..."
76
+ export AWS_SECRET_KEY="secret..."
77
+ dremio sync push
78
+ ```
79
+
80
+ If a variable is missing, the CLI will warn you and keep the literal string (which leads to auth failure), protecting you from accidental commits of unexpanded secrets.
@@ -0,0 +1,78 @@
1
+
2
+ # Managing Tables in Dremio-as-Code
3
+
4
+ DAC supports managing Physical Iceberg Tables and standard Physical Datasets.
5
+
6
+ ## 1. Iceberg Tables (`type: ICEBERG_TABLE`)
7
+
8
+ You can use DAC to manage the lifecycle of Dremio-native Iceberg tables (e.g., in Arctic, Nessie, or S3). This allows for "Lightweight ETL" where every `push` triggers an SQL update script.
9
+
10
+ ### YAML Schema
11
+
12
+ ```yaml
13
+ name: "app_events"
14
+ type: ICEBERG_TABLE
15
+ path: ["dremio", "etl", "app_events"]
16
+
17
+ # Run ONE-TIME if table doesn't exist
18
+ create_sql: |
19
+ CREATE TABLE dremio.etl.app_events (
20
+ id VARCHAR,
21
+ event_time TIMESTAMP,
22
+ payload VARCHAR
23
+ ) PARTITION BY (event_time)
24
+
25
+ # Run EVERY PUSH (if table exists)
26
+ update_sql: |
27
+ MERGE INTO dremio.etl.app_events t
28
+ USING source.raw_events s
29
+ ON t.id = s.id
30
+ WHEN MATCHED THEN UPDATE SET t.payload = s.payload
31
+ WHEN NOT MATCHED THEN INSERT VALUES (s.id, s.event_time, s.payload)
32
+
33
+ # Standard features
34
+ tags: ["events", "etl"]
35
+ governance: ...
36
+ reflections: ...
37
+ ```
38
+
39
+ ### Workflow
40
+ 1. **First Push**: CLI detects missing table. executes `create_sql`.
41
+ 2. **Subsequent Pushes**: CLI detects existing table. Executes `update_sql`.
42
+
43
+ ### Pulling Iceberg Tables
44
+ `dremio sync pull` will generate the YAML for existing Iceberg tables.
45
+ *Limitation*: The CLI cannot reconstruct the `create_sql` or `update_sql` logic. The fields will be generated as placeholders or comments.
46
+
47
+ ## 2. Physical Datasets (`type: PHYSICAL_DATASET`)
48
+
49
+ For external files (S3 Parquet/CSV) or RDBMS tables that are read-only to Dremio, use `PHYSICAL_DATASET`. DAC manages their **metadata** (Governance, Reflections, Wiki), not their data lifecycle.
50
+
51
+ ### YAML Schema
52
+
53
+ ```yaml
54
+ name: "raw_customers"
55
+ type: PHYSICAL_DATASET
56
+ path: ["s3-source", "bucket", "customers.parquet"]
57
+
58
+ # Metadata
59
+ tags: ["raw", "source"]
60
+ description: "docs/raw_customers.md"
61
+
62
+ # Format (Optional - for auto-promotion)
63
+ format:
64
+ type: "Parquet"
65
+
66
+ # Governance
67
+ access_control:
68
+ roles:
69
+ - name: "analysts"
70
+ privileges: ["SELECT"]
71
+ reflections:
72
+ - name: "raw_ref"
73
+ type: "RAW"
74
+ ```
75
+
76
+ ### Workflow
77
+ - **Push**: Ensures the dataset is promoted/exists. Applies tags, wiki, grants, and reflections.
78
+ - **Pull**: Generates YAML for existing promoted datasets in the scope.
@@ -0,0 +1,54 @@
1
+
2
+ # Data Validations in Dremio-as-Code
3
+
4
+ Ensure data quality and integrity by defining SQL assertions that run after every `sync push`.
5
+
6
+ ## Overview
7
+
8
+ Validations are SQL queries paired with a condition. The CLI executes the query, fetches the scalar result (first column of first row), and compares it against your condition.
9
+
10
+ ## YAML Schema
11
+
12
+ Add a `validations` block to any dataset YAML (`VIRTUAL_DATASET` or `ICEBERG_TABLE`).
13
+
14
+ ```yaml
15
+ name: "mart_revenue"
16
+ type: VIRTUAL_DATASET
17
+ path: ["dremio", "finance", "mart_revenue"]
18
+ sql: "SELECT ..."
19
+
20
+ validations:
21
+ # Check 1: Non-zero rows
22
+ - name: "row_count"
23
+ sql: "SELECT count(*) FROM dremio.finance.mart_revenue"
24
+ condition: "gt 0"
25
+
26
+ # Check 2: No null regions
27
+ - name: "no_null_regions"
28
+ sql: "SELECT count(*) FROM dremio.finance.mart_revenue WHERE region IS NULL"
29
+ condition: "eq 0"
30
+
31
+ # Check 3: Total is positive
32
+ - name: "positive_revenue"
33
+ sql: "SELECT min(total_revenue) FROM dremio.finance.mart_revenue"
34
+ condition: "gte 0"
35
+ ```
36
+
37
+ ## Supported Conditions
38
+
39
+ - `eq {val}`: Equal to value
40
+ - `neq {val}`: Not equal to value
41
+ - `gt {val}`: Greater than value
42
+ - `lt {val}`: Less than value
43
+ - `gte {val}`: Greater than or equal to value
44
+ - `lte {val}`: Less than or equal to value
45
+
46
+ ## Workflow
47
+
48
+ 1. Sync (Create/Update View/Table).
49
+ 2. Wait for completion.
50
+ 3. **Run Validations**:
51
+ - Execute Check SQL.
52
+ - Fetch Result.
53
+ - Evaluate Condition.
54
+ - Log `[PASS]` or `[FAIL]`.
@@ -0,0 +1,69 @@
1
+
2
+ from typing import List, Dict, Set, Any
3
+ from collections import defaultdict, deque
4
+
5
+ class DependencyGraph:
6
+ """Manages dependency graph and topological sorting."""
7
+
8
+ def __init__(self, items: List[Dict[str, Any]]):
9
+ """Initialize with list of items (dicts).
10
+
11
+ Each item usually has:
12
+ - name: str (unique identifier if possible, or we assume name matches dependency ref)
13
+ - dependencies: List[str]
14
+ """
15
+ self.items = items
16
+ self.item_map = {item.get("name"): item for item in items if item.get("name")}
17
+
18
+ def get_execution_order(self) -> List[Dict[str, Any]]:
19
+ """Get topological sort of items.
20
+
21
+ Returns:
22
+ List of items in execution order.
23
+
24
+ Raises:
25
+ ValueError: If cycle detected.
26
+ """
27
+ # Build graph
28
+ adj = defaultdict(list)
29
+ in_degree = defaultdict(int)
30
+
31
+ # Initialize in_degree for all known items
32
+ for name in self.item_map:
33
+ in_degree[name] = 0
34
+
35
+ # Create edges
36
+ for item in self.items:
37
+ u = item.get("name")
38
+ if not u: continue
39
+
40
+ deps = item.get("dependencies", [])
41
+ if not isinstance(deps, list): deps = []
42
+
43
+ for v in deps:
44
+ # If dependency exists in our local set, add edge v -> u (v must come before u)
45
+ if v in self.item_map:
46
+ adj[v].append(u)
47
+ in_degree[u] += 1
48
+ # If dependency is external (not in local set), we ignore it and assume it exists remotely
49
+
50
+ # Kahn's Algorithm
51
+ queue = deque([node for node in self.item_map if in_degree[node] == 0])
52
+ result = []
53
+
54
+ while queue:
55
+ u = queue.popleft()
56
+ result.append(self.item_map[u])
57
+
58
+ for v in adj[u]:
59
+ in_degree[v] -= 1
60
+ if in_degree[v] == 0:
61
+ queue.append(v)
62
+
63
+ # Cycle Check
64
+ if len(result) != len(self.item_map):
65
+ # Find missing nodes
66
+ missing = set(self.item_map.keys()) - set(item["name"] for item in result)
67
+ raise ValueError(f"Circular dependency detected involving: {missing}")
68
+
69
+ return result
@@ -57,28 +57,63 @@ class LocalState:
57
57
 
58
58
  def scan_filesystem(self) -> Dict[str, Dict[str, Any]]:
59
59
  """
60
- Scans the root_dir for .yaml and .sql files to build a picture of local state.
60
+ Scans the root_dir (and children) for .yaml files to build resource state.
61
61
  Returns a dict of resource definitions found on disk.
62
62
  """
63
63
  local_resources = {}
64
64
  for root, dirs, files in os.walk(self.root_path):
65
- # skipping .git, .dremio_state.json, etc is handled by not parsing them
65
+ # skips
66
+ if ".git" in dirs: dirs.remove(".git")
67
+
66
68
  for file in files:
67
69
  if file.endswith(".yaml") and file != "dremio.yaml":
68
70
  full_path = os.path.join(root, file)
71
+ rel_dir = os.path.relpath(root, self.root_path)
72
+ if rel_dir == ".": rel_dir = ""
73
+
69
74
  try:
70
75
  with open(full_path, 'r') as f:
71
76
  data = yaml.safe_load(f)
72
- # We expect 'type' and 'path' in the yaml
73
- if data and "type" in data and "path" in data:
74
- path_key = ".".join(data["path"])
75
- local_resources[path_key] = data
76
- # If there is a sql file, read it
77
- if "sql_file" in data:
78
- sql_path = os.path.join(root, data["sql_file"])
79
- if os.path.exists(sql_path):
80
- with open(sql_path, 'r') as sql_f:
81
- data["sql_content"] = sql_f.read()
77
+
78
+ if not data or "name" not in data or "type" not in data:
79
+ continue
80
+
81
+ # Determine full path key
82
+ # Logic: if 'path' explicit in YAML, use it.
83
+ # Else: derive from valid dremio.yaml scope + subdirs
84
+ resource_path = []
85
+ if "path" in data:
86
+ resource_path = data["path"]
87
+ else:
88
+ # We need context/scope to know the root prefix.
89
+ # For now, we store just the name, assuming sync logic resolves parent
90
+ # OR ideally: we require 'path' OR we inject it from outside.
91
+ # Let's rely on 'name' key grouping for now, but sync logic needs full path.
92
+ # Store relative path for now.
93
+ resource_path = [data["name"]]
94
+
95
+ path_key = ".".join(resource_path)
96
+
97
+ # Handle SQL
98
+ if "sql" in data:
99
+ data["sql_content"] = data["sql"]
100
+ elif "sql_content" in data:
101
+ pass # Already there
102
+
103
+ # Handle Wiki (description file)
104
+ desc = data.get("description", "")
105
+ if desc and desc.endswith(".md"):
106
+ md_path = os.path.join(root, desc)
107
+ if os.path.exists(md_path):
108
+ with open(md_path, 'r') as md_f:
109
+ data["description"] = md_f.read()
110
+ else:
111
+ print(f"Warning: Wiki file {md_path} referenced but not found.")
112
+
113
+ # Store dependencies and tags implicitly (passed as is)
114
+
115
+ local_resources[path_key] = data
116
+
82
117
  except Exception as e:
83
118
  print(f"Error reading {full_path}: {e}")
84
119
  return local_resources