datasecops-cli 0.4.4__tar.gz → 0.4.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/CHANGELOG.md +19 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/PKG-INFO +1 -1
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/pyproject.toml +1 -1
- datasecops_cli-0.4.6/src/datasecops_cli/__init__.py +1 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/main.py +7 -1
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/menus/development.py +6 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/menus/downloads.py +4 -2
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/bootstrap_service.py +5 -2
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/dbt_project_generator.py +19 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/download_service.py +61 -146
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/snowflake_service.py +5 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_main.py +118 -1
- datasecops_cli-0.4.4/src/datasecops_cli/__init__.py +0 -1
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/.github/workflows/auto-tag.yml +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/.github/workflows/publish-cli.yml +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/.gitignore +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/DEVELOPMENT.md +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/LICENSE +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/README.md +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/docs/getting-started.md +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/docs/legacy.md +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/docs/legacy_plan_of_action.md +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/docs/mcp-server.md +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/mcp-servers.json +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/setup.ps1 +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/setup.sh +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/config.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/menus/__init__.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/menus/configuration.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/menus/git_operations.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/models/__init__.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/models/git_helpers.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/models/project_config.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/__init__.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/dbt_runner.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/directory_scaffolder.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/git_service.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/linting_service.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/skill_service.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/upstream_service.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/utilities/__init__.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/utilities/display.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/utilities/file_utils.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_mcp/__init__.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_mcp/__main__.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_mcp/connection.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_mcp/server.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/__init__.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_config.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_file_utils.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_models.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_version.py +0 -0
- {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_yaml_utils.py +0 -0
|
@@ -2,6 +2,25 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to the DataSecOps CLI are documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.4.6] - 2026-05-18
|
|
6
|
+
|
|
7
|
+
### Changed
|
|
8
|
+
|
|
9
|
+
- **SQLFluff config now fetched as rendered INI from native app** — `download_sqlfluff_config()` calls the new `api.get_sqlfluff_config_file()` stored procedure which returns a ready-to-use `.sqlfluff` file, matching exactly what the native app UI displays. Removes client-side INI rendering logic.
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **Re-download .sqlfluff option in lint menu** — new option `[6] refresh config` in the SQLFluff linting menu to re-download the `.sqlfluff` configuration from the framework at any time, even if the file already exists locally.
|
|
14
|
+
- **`get_sqlfluff_config_file()` method** added to `SnowflakeService` for calling the new native app procedure.
|
|
15
|
+
|
|
16
|
+
## [0.4.5] - 2026-05-18
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- **Package downloads now respect profile selection** — `datasecops download packages` and the interactive downloads menu now only include dbt packages enabled on the active profile, instead of downloading all packages from the global catalog. This matches the native app's Streamlit UI behavior.
|
|
21
|
+
- **Upstream package exclusion** — packages already provided by upstream projects are automatically excluded from `packages.yml`, preventing duplicate dependencies when using multi-project setups.
|
|
22
|
+
- **`generate_packages_yml()` filtering** — the dbt project generator now also filters packages by profile, fixing the same issue in the bootstrap/init path.
|
|
23
|
+
|
|
5
24
|
## [0.4.4] - 2026-05-18
|
|
6
25
|
|
|
7
26
|
### Added
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.4.6"
|
|
@@ -565,7 +565,10 @@ def _run_download(config: Config, items: list[str],
|
|
|
565
565
|
if not download_service.download_pipelines(platform=platform, profile_name=config.profile_name):
|
|
566
566
|
failed = True
|
|
567
567
|
elif item == "packages":
|
|
568
|
-
if not download_service.download_dbt_packages(
|
|
568
|
+
if not download_service.download_dbt_packages(
|
|
569
|
+
config.dbt_project_dir, profile=config.profile,
|
|
570
|
+
all_profiles=config.all_profiles
|
|
571
|
+
):
|
|
569
572
|
failed = True
|
|
570
573
|
elif item == "macros":
|
|
571
574
|
if not download_service.download_macros(config.profile_name, config.dbt_project_dir):
|
|
@@ -704,6 +707,7 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
|
|
|
704
707
|
project_settings=config.project_settings,
|
|
705
708
|
profile=config.profile,
|
|
706
709
|
source_control=config.source_control,
|
|
710
|
+
all_profiles=config.all_profiles,
|
|
707
711
|
)
|
|
708
712
|
dl_menu.show()
|
|
709
713
|
|
|
@@ -722,6 +726,7 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
|
|
|
722
726
|
project_dir=config.project_dir,
|
|
723
727
|
project_settings=config.project_settings,
|
|
724
728
|
profile=config.profile,
|
|
729
|
+
all_profiles=config.all_profiles,
|
|
725
730
|
)
|
|
726
731
|
bootstrap.run(platform=platform, install_skills=install_skills,
|
|
727
732
|
skill_targets=skill_targets, run_deps=run_deps)
|
|
@@ -784,6 +789,7 @@ def _run_bootstrap(config: Config):
|
|
|
784
789
|
project_dir=config.project_dir,
|
|
785
790
|
project_settings=config.project_settings,
|
|
786
791
|
profile=config.profile,
|
|
792
|
+
all_profiles=config.all_profiles,
|
|
787
793
|
)
|
|
788
794
|
success = bootstrap.run(platform=platform, install_skills=install_skills,
|
|
789
795
|
skill_targets=skill_targets, run_deps=run_deps)
|
|
@@ -176,6 +176,7 @@ class DevelopmentMenu:
|
|
|
176
176
|
menu_option(3, "find all - Lint all SQL files")
|
|
177
177
|
menu_option(4, "fix all - Fix all SQL files")
|
|
178
178
|
menu_option(5, "specific - Lint specific file")
|
|
179
|
+
menu_option(6, "refresh config- Re-download .sqlfluff from framework")
|
|
179
180
|
menu_option(0, "back - Return to development menu")
|
|
180
181
|
option = get_input_number("Choose an option: ")
|
|
181
182
|
if option in (1, 2, 3, 4, 5) and not self._ensure_sqlfluff_config():
|
|
@@ -195,6 +196,11 @@ class DevelopmentMenu:
|
|
|
195
196
|
path = get_input_string("Enter SQL file path (e.g. models/staging/stg_orders.sql): ")
|
|
196
197
|
if path != "0":
|
|
197
198
|
self.linting.lint_file(file_path=path, fix=False)
|
|
199
|
+
elif option == 6:
|
|
200
|
+
if not self.downloads:
|
|
201
|
+
error_line("Download service not available")
|
|
202
|
+
else:
|
|
203
|
+
self.downloads.download_sqlfluff_config(profiles_dir=self.profiles_dir, dbt_project_dir=self.linting.project_dir)
|
|
198
204
|
complete_action()
|
|
199
205
|
|
|
200
206
|
def _autofix_menu(self) -> None:
|
|
@@ -19,7 +19,7 @@ class DownloadsMenu:
|
|
|
19
19
|
def __init__(self, download_service: DownloadService, skill_service: SkillService,
|
|
20
20
|
dbt_runner: DbtRunner, profile_name: str, dbt_project_dir: Path,
|
|
21
21
|
project_settings: ProjectSettings = None, profile: ProjectProfile = None,
|
|
22
|
-
source_control: SourceControl = None):
|
|
22
|
+
source_control: SourceControl = None, all_profiles: list[ProjectProfile] = None):
|
|
23
23
|
self.downloads = download_service
|
|
24
24
|
self.skills = skill_service
|
|
25
25
|
self.dbt = dbt_runner
|
|
@@ -28,6 +28,7 @@ class DownloadsMenu:
|
|
|
28
28
|
self.project_settings = project_settings
|
|
29
29
|
self.profile = profile
|
|
30
30
|
self.source_control = source_control
|
|
31
|
+
self.all_profiles = all_profiles
|
|
31
32
|
|
|
32
33
|
def show(self) -> None:
|
|
33
34
|
self._menu()
|
|
@@ -46,7 +47,8 @@ class DownloadsMenu:
|
|
|
46
47
|
complete_action()
|
|
47
48
|
elif option == 3:
|
|
48
49
|
display_action_header("Download dbt Packages")
|
|
49
|
-
self.downloads.download_dbt_packages(self.dbt_project_dir
|
|
50
|
+
self.downloads.download_dbt_packages(self.dbt_project_dir, profile=self.profile,
|
|
51
|
+
all_profiles=self.all_profiles)
|
|
50
52
|
if get_input_true_false("Run dbt deps now?"):
|
|
51
53
|
self.dbt.deps()
|
|
52
54
|
complete_action()
|
{datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/bootstrap_service.py
RENAMED
|
@@ -14,11 +14,13 @@ class BootstrapService:
|
|
|
14
14
|
"""Bootstraps a new dbt project with all framework configuration."""
|
|
15
15
|
|
|
16
16
|
def __init__(self, snowflake_service: SnowflakeService, project_dir: Path,
|
|
17
|
-
project_settings: ProjectSettings, profile: ProjectProfile
|
|
17
|
+
project_settings: ProjectSettings, profile: ProjectProfile,
|
|
18
|
+
all_profiles: list[ProjectProfile] = None):
|
|
18
19
|
self.sf = snowflake_service
|
|
19
20
|
self.project_dir = project_dir
|
|
20
21
|
self.project_settings = project_settings
|
|
21
22
|
self.profile = profile
|
|
23
|
+
self.all_profiles = all_profiles or []
|
|
22
24
|
self.download_service = DownloadService(snowflake_service, project_dir)
|
|
23
25
|
self.skill_service = SkillService(snowflake_service)
|
|
24
26
|
|
|
@@ -92,7 +94,8 @@ class BootstrapService:
|
|
|
92
94
|
# Step 5: Generate packages.yml
|
|
93
95
|
info_line("")
|
|
94
96
|
info_line("[5] Generating packages.yml...")
|
|
95
|
-
if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile
|
|
97
|
+
if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile,
|
|
98
|
+
all_profiles=self.all_profiles):
|
|
96
99
|
steps_passed += 1
|
|
97
100
|
else:
|
|
98
101
|
info_line(" (skipped - no dbt packages config in native app)")
|
{datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/dbt_project_generator.py
RENAMED
|
@@ -165,17 +165,36 @@ def generate_profiles_yml(
|
|
|
165
165
|
def generate_packages_yml(
|
|
166
166
|
profile: ProjectProfile,
|
|
167
167
|
packages_config: dict,
|
|
168
|
+
all_profiles: list[ProjectProfile] = None,
|
|
168
169
|
) -> str:
|
|
169
170
|
"""Generate packages.yml from profile's packages and upstream projects.
|
|
170
171
|
|
|
171
172
|
Args:
|
|
172
173
|
profile: The project profile.
|
|
173
174
|
packages_config: The DBT_PACKAGES framework config.
|
|
175
|
+
all_profiles: All project profiles (used for upstream package exclusion).
|
|
174
176
|
|
|
175
177
|
Returns:
|
|
176
178
|
The YAML content as a string.
|
|
177
179
|
"""
|
|
178
180
|
packages = packages_config.get("packages", [])
|
|
181
|
+
|
|
182
|
+
# Filter to only packages enabled on the profile
|
|
183
|
+
if profile.dbt_packages:
|
|
184
|
+
from datasecops_cli.services.download_service import _collect_upstream_packages
|
|
185
|
+
|
|
186
|
+
enabled = set(profile.dbt_packages)
|
|
187
|
+
|
|
188
|
+
# Exclude packages already provided by upstream projects
|
|
189
|
+
if profile.upstream_projects and all_profiles:
|
|
190
|
+
profile_map = {p.profile_name: p for p in all_profiles}
|
|
191
|
+
upstream_pkgs = _collect_upstream_packages(
|
|
192
|
+
profile.upstream_projects, profile_map
|
|
193
|
+
)
|
|
194
|
+
enabled -= upstream_pkgs
|
|
195
|
+
|
|
196
|
+
packages = [p for p in packages if p.get("name") in enabled]
|
|
197
|
+
|
|
179
198
|
pkg_list: list[dict[str, Any]] = []
|
|
180
199
|
|
|
181
200
|
for pkg in packages:
|
{datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/download_service.py
RENAMED
|
@@ -10,164 +10,62 @@ from datasecops_cli.utilities.display import info_line, success_line, error_line
|
|
|
10
10
|
from datasecops_cli.utilities.file_utils import write_file, ensure_dir
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
def _collect_upstream_packages(
|
|
14
|
+
upstream_names: list[str],
|
|
15
|
+
profile_map: dict[str, ProjectProfile],
|
|
16
|
+
visited: set[str] | None = None,
|
|
17
|
+
) -> set[str]:
|
|
18
|
+
"""Recursively collect dbt package names from all upstream projects."""
|
|
19
|
+
if visited is None:
|
|
20
|
+
visited = set()
|
|
21
|
+
result: set[str] = set()
|
|
22
|
+
for name in upstream_names:
|
|
23
|
+
if name in visited:
|
|
24
|
+
continue
|
|
25
|
+
visited.add(name)
|
|
26
|
+
upstream = profile_map.get(name)
|
|
27
|
+
if upstream:
|
|
28
|
+
result.update(upstream.dbt_packages)
|
|
29
|
+
result.update(
|
|
30
|
+
_collect_upstream_packages(
|
|
31
|
+
upstream.upstream_projects or [], profile_map, visited
|
|
32
|
+
)
|
|
33
|
+
)
|
|
34
|
+
return result
|
|
35
|
+
|
|
36
|
+
|
|
13
37
|
class DownloadService:
|
|
14
38
|
"""Downloads configurations from the native app."""
|
|
15
39
|
|
|
16
|
-
# Maps rule codes to their sqlfluff INI section names.
|
|
17
|
-
RULE_SECTION_MAP: dict[str, str] = {
|
|
18
|
-
# Aliasing
|
|
19
|
-
"AL01": "aliasing.table", "AL02": "aliasing.column", "AL03": "aliasing.expression",
|
|
20
|
-
"AL04": "aliasing.unique.table", "AL05": "aliasing.unused", "AL06": "aliasing.length",
|
|
21
|
-
"AL07": "aliasing.forbid", "AL08": "aliasing.unique.column", "AL09": "aliasing.self_alias",
|
|
22
|
-
# Ambiguous
|
|
23
|
-
"AM01": "ambiguous.distinct", "AM02": "ambiguous.union", "AM03": "ambiguous.order_by",
|
|
24
|
-
"AM04": "ambiguous.column_count", "AM05": "ambiguous.join",
|
|
25
|
-
"AM06": "ambiguous.column_references", "AM07": "ambiguous.set_columns",
|
|
26
|
-
"AM08": "ambiguous.union_type",
|
|
27
|
-
# Capitalisation
|
|
28
|
-
"CP01": "capitalisation.keywords", "CP02": "capitalisation.identifiers",
|
|
29
|
-
"CP03": "capitalisation.functions", "CP04": "capitalisation.literals",
|
|
30
|
-
"CP05": "capitalisation.types",
|
|
31
|
-
# Convention
|
|
32
|
-
"CV01": "convention.not_equal", "CV02": "convention.coalesce",
|
|
33
|
-
"CV03": "convention.select_trailing_comma", "CV04": "convention.count_rows",
|
|
34
|
-
"CV05": "convention.is_null", "CV06": "convention.terminator",
|
|
35
|
-
"CV07": "convention.statement_brackets", "CV08": "convention.left_join",
|
|
36
|
-
"CV09": "convention.blocked_words", "CV10": "convention.quoted_literals",
|
|
37
|
-
"CV11": "convention.casting_style", "CV12": "convention.semicolon",
|
|
38
|
-
# Jinja
|
|
39
|
-
"JJ01": "jinja.padding",
|
|
40
|
-
# Layout
|
|
41
|
-
"LT01": "layout.spacing", "LT02": "layout.indent", "LT03": "layout.operators",
|
|
42
|
-
"LT04": "layout.commas", "LT05": "layout.long_lines", "LT06": "layout.functions",
|
|
43
|
-
"LT07": "layout.cte_bracket", "LT08": "layout.cte_newline",
|
|
44
|
-
"LT09": "layout.select_targets", "LT10": "layout.select_modifiers",
|
|
45
|
-
"LT11": "layout.set_operators", "LT12": "layout.end-of-file",
|
|
46
|
-
"LT13": "layout.start_of_file", "LT14": "layout.one_line", "LT15": "layout.newlines",
|
|
47
|
-
# References
|
|
48
|
-
"RF01": "references.from", "RF02": "references.qualification",
|
|
49
|
-
"RF03": "references.consistent", "RF04": "references.keywords",
|
|
50
|
-
"RF05": "references.special_chars", "RF06": "references.quoting",
|
|
51
|
-
# Structure
|
|
52
|
-
"ST01": "structure.else_null", "ST02": "structure.simple_case",
|
|
53
|
-
"ST03": "structure.unused_cte", "ST04": "structure.nested_case",
|
|
54
|
-
"ST05": "structure.subquery", "ST06": "structure.column_order",
|
|
55
|
-
"ST07": "structure.using", "ST08": "structure.distinct",
|
|
56
|
-
"ST09": "structure.join_condition_order", "ST10": "structure.cte_definition_order",
|
|
57
|
-
"ST11": "structure.test_query",
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
# Core sub-keys that map to specific INI sections.
|
|
61
|
-
CORE_SECTION_MAP: dict[str, str] = {
|
|
62
|
-
"sqlfluff": "sqlfluff",
|
|
63
|
-
"dbt": "sqlfluff:templater:dbt",
|
|
64
|
-
"jinja": "sqlfluff:templater:jinja",
|
|
65
|
-
"templater": "sqlfluff:templater",
|
|
66
|
-
}
|
|
67
|
-
|
|
68
40
|
def __init__(self, snowflake_service: SnowflakeService, project_dir: Path):
|
|
69
41
|
self.sf = snowflake_service
|
|
70
42
|
self.project_dir = project_dir
|
|
71
|
-
|
|
72
|
-
# Options to strip from specific core sections (not valid sqlfluff config keys).
|
|
73
|
-
CORE_STRIP_OPTIONS: dict[str, set[str]] = {
|
|
74
|
-
"dbt": {"dbt_skip_compilation_error"},
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
@staticmethod
|
|
78
|
-
def _format_value(val) -> str:
|
|
79
|
-
"""Format a JSON value for sqlfluff INI output."""
|
|
80
|
-
if isinstance(val, bool):
|
|
81
|
-
return str(val)
|
|
82
|
-
if isinstance(val, list):
|
|
83
|
-
if not val:
|
|
84
|
-
return "None"
|
|
85
|
-
return ", ".join(str(v) for v in val)
|
|
86
|
-
if val is None or val == "":
|
|
87
|
-
return "None"
|
|
88
|
-
return str(val)
|
|
89
|
-
|
|
90
|
-
@staticmethod
|
|
91
|
-
def _emit_section(lines: list[str], header: str, options: dict) -> None:
|
|
92
|
-
"""Append an INI section with its options to lines."""
|
|
93
|
-
lines.append(f"[{header}]")
|
|
94
|
-
for key, val in options.items():
|
|
95
|
-
lines.append(f"{key} = {DownloadService._format_value(val)}")
|
|
96
|
-
lines.append("")
|
|
97
43
|
|
|
98
44
|
def download_sqlfluff_config(self, profiles_dir: str = None, dbt_project_dir: Path = None) -> bool:
|
|
99
45
|
info_line("Downloading SQLFluff configuration...")
|
|
100
|
-
|
|
101
|
-
if not
|
|
46
|
+
content = self.sf.get_sqlfluff_config_file()
|
|
47
|
+
if not content:
|
|
102
48
|
error_line("No SQLFluff configuration found in native app")
|
|
103
49
|
return False
|
|
104
|
-
|
|
105
|
-
lines: list[str] = []
|
|
106
50
|
|
|
107
|
-
#
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
if
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
if opts:
|
|
125
|
-
self._emit_section(lines, section_header, opts)
|
|
51
|
+
# If profiles_dir is specified, ensure it's set in the [sqlfluff:templater:dbt] section
|
|
52
|
+
if profiles_dir and "[sqlfluff:templater:dbt]" in content:
|
|
53
|
+
import re
|
|
54
|
+
# Replace existing profiles_dir line if present, otherwise insert after header
|
|
55
|
+
if re.search(r"^profiles_dir\s*=.*$", content, re.MULTILINE):
|
|
56
|
+
content = re.sub(
|
|
57
|
+
r"^profiles_dir\s*=.*$",
|
|
58
|
+
f"profiles_dir = {profiles_dir}",
|
|
59
|
+
content,
|
|
60
|
+
count=1,
|
|
61
|
+
flags=re.MULTILINE,
|
|
62
|
+
)
|
|
63
|
+
else:
|
|
64
|
+
content = content.replace(
|
|
65
|
+
"[sqlfluff:templater:dbt]",
|
|
66
|
+
f"[sqlfluff:templater:dbt]\nprofiles_dir = {profiles_dir}",
|
|
67
|
+
)
|
|
126
68
|
|
|
127
|
-
# --- Indentation ---
|
|
128
|
-
indentation = raw.get("indentation", {})
|
|
129
|
-
for _code, entry in indentation.items():
|
|
130
|
-
if isinstance(entry, dict) and entry.get("enabled", True):
|
|
131
|
-
opts = entry.get("options", {})
|
|
132
|
-
if opts:
|
|
133
|
-
self._emit_section(lines, "sqlfluff:indentation", opts)
|
|
134
|
-
|
|
135
|
-
# --- Layout sections ([sqlfluff:layout:type:<code>]) ---
|
|
136
|
-
layout = raw.get("layout", {})
|
|
137
|
-
for code, entry in layout.items():
|
|
138
|
-
if not isinstance(entry, dict) or not entry.get("enabled", True):
|
|
139
|
-
continue
|
|
140
|
-
opts = entry.get("options", {})
|
|
141
|
-
if opts:
|
|
142
|
-
self._emit_section(lines, f"sqlfluff:layout:type:{code}", opts)
|
|
143
|
-
|
|
144
|
-
# --- Global rules ([sqlfluff:rules]) ---
|
|
145
|
-
rules = raw.get("rules", {})
|
|
146
|
-
for _code, entry in rules.items():
|
|
147
|
-
if isinstance(entry, dict) and entry.get("enabled", True):
|
|
148
|
-
opts = entry.get("options", {})
|
|
149
|
-
if opts:
|
|
150
|
-
self._emit_section(lines, "sqlfluff:rules", opts)
|
|
151
|
-
|
|
152
|
-
# --- Rule bundle sections (rules_aliasing, rules_capitalisation, etc.) ---
|
|
153
|
-
for section_key, entries in raw.items():
|
|
154
|
-
if not section_key.startswith("rules_") or not isinstance(entries, dict):
|
|
155
|
-
continue
|
|
156
|
-
for code, entry in entries.items():
|
|
157
|
-
if not isinstance(entry, dict) or not entry.get("enabled", True):
|
|
158
|
-
continue
|
|
159
|
-
section_name = self.RULE_SECTION_MAP.get(code)
|
|
160
|
-
if not section_name:
|
|
161
|
-
continue
|
|
162
|
-
opts = dict(entry.get("options", {}))
|
|
163
|
-
# For aliasing.length, 0 means "no limit" which sqlfluff expects as None
|
|
164
|
-
if section_name == "aliasing.length":
|
|
165
|
-
for len_key in ("min_alias_length", "max_alias_length"):
|
|
166
|
-
if len_key in opts and opts[len_key] == 0:
|
|
167
|
-
opts[len_key] = None
|
|
168
|
-
self._emit_section(lines, f"sqlfluff:rules:{section_name}", opts)
|
|
169
|
-
|
|
170
|
-
content = "\n".join(lines)
|
|
171
69
|
dest = (dbt_project_dir or self.project_dir) / ".sqlfluff"
|
|
172
70
|
write_file(dest, content)
|
|
173
71
|
success_line(f"SQLFluff config written to {dest}")
|
|
@@ -279,7 +177,8 @@ class DownloadService:
|
|
|
279
177
|
success_line(f"Downloaded {count} script(s) to {scripts_dir}")
|
|
280
178
|
return True
|
|
281
179
|
|
|
282
|
-
def download_dbt_packages(self, dbt_project_dir: Path, profile: ProjectProfile = None
|
|
180
|
+
def download_dbt_packages(self, dbt_project_dir: Path, profile: ProjectProfile = None,
|
|
181
|
+
all_profiles: list[ProjectProfile] = None) -> bool:
|
|
283
182
|
info_line("Downloading dbt package versions...")
|
|
284
183
|
raw = self.sf.get_framework_config("DBT_PACKAGES")
|
|
285
184
|
if not raw:
|
|
@@ -287,6 +186,22 @@ class DownloadService:
|
|
|
287
186
|
return False
|
|
288
187
|
|
|
289
188
|
packages = raw.get("packages", [])
|
|
189
|
+
|
|
190
|
+
# Filter to only packages enabled on the active profile
|
|
191
|
+
if profile and profile.dbt_packages:
|
|
192
|
+
enabled = set(profile.dbt_packages)
|
|
193
|
+
|
|
194
|
+
# Exclude packages already provided by upstream projects
|
|
195
|
+
if profile.upstream_projects and all_profiles:
|
|
196
|
+
profile_map = {p.profile_name: p for p in all_profiles}
|
|
197
|
+
upstream_pkgs = _collect_upstream_packages(
|
|
198
|
+
profile.upstream_projects, profile_map
|
|
199
|
+
)
|
|
200
|
+
enabled -= upstream_pkgs
|
|
201
|
+
|
|
202
|
+
packages = [p for p in packages if p.get("name") in enabled]
|
|
203
|
+
info_line(f" Filtered to {len(packages)} package(s) for profile '{profile.profile_name}'")
|
|
204
|
+
|
|
290
205
|
pkg_list = []
|
|
291
206
|
for pkg in packages:
|
|
292
207
|
source = pkg.get("source", "git")
|
{datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/snowflake_service.py
RENAMED
|
@@ -81,3 +81,8 @@ class SnowflakeService:
|
|
|
81
81
|
|
|
82
82
|
def get_support_contacts(self) -> list:
|
|
83
83
|
return self.call_procedure("get_support_contacts") or []
|
|
84
|
+
|
|
85
|
+
def get_sqlfluff_config_file(self) -> str:
|
|
86
|
+
"""Get the rendered .sqlfluff config INI text from the native app."""
|
|
87
|
+
result = self.call_procedure("get_sqlfluff_config_file")
|
|
88
|
+
return result if isinstance(result, str) else ""
|
|
@@ -5,6 +5,7 @@ from pathlib import Path
|
|
|
5
5
|
|
|
6
6
|
from datasecops_cli.main import _build_parser, _run_download, DOWNLOAD_ITEMS
|
|
7
7
|
from datasecops_cli.config import Config
|
|
8
|
+
from datasecops_cli.models.project_config import ProjectProfile
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class TestBuildParser:
|
|
@@ -64,6 +65,11 @@ class TestRunDownload:
|
|
|
64
65
|
config.project_dir = tmp_path
|
|
65
66
|
config.dbt_project_dir = tmp_path
|
|
66
67
|
config.profile_name = "test_profile"
|
|
68
|
+
config.profile = ProjectProfile(
|
|
69
|
+
profile_name="test_profile",
|
|
70
|
+
dbt_packages=["dbt_utils", "dbt_expectations"],
|
|
71
|
+
)
|
|
72
|
+
config.all_profiles = [config.profile]
|
|
67
73
|
# source_control defaults to GitHub
|
|
68
74
|
return config
|
|
69
75
|
|
|
@@ -119,7 +125,9 @@ class TestRunDownload:
|
|
|
119
125
|
_run_download(config, ["packages"])
|
|
120
126
|
|
|
121
127
|
assert exc.value.code == 0
|
|
122
|
-
mock_ds.download_dbt_packages.assert_called_once_with(
|
|
128
|
+
mock_ds.download_dbt_packages.assert_called_once_with(
|
|
129
|
+
tmp_path, profile=config.profile, all_profiles=config.all_profiles
|
|
130
|
+
)
|
|
123
131
|
|
|
124
132
|
@patch("datasecops_cli.main._connect_and_load")
|
|
125
133
|
def test_download_macros(self, mock_connect, tmp_path):
|
|
@@ -300,3 +308,112 @@ class TestRunDownload:
|
|
|
300
308
|
_run_download(config, ["install-dbt"])
|
|
301
309
|
|
|
302
310
|
assert exc.value.code == 1
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
class TestDownloadPackageFiltering:
|
|
314
|
+
"""Tests for dbt package filtering by profile."""
|
|
315
|
+
|
|
316
|
+
def test_download_dbt_packages_filters_by_profile(self, tmp_path):
|
|
317
|
+
"""Only packages in profile.dbt_packages should appear in packages.yml."""
|
|
318
|
+
from datasecops_cli.services.download_service import DownloadService
|
|
319
|
+
|
|
320
|
+
mock_sf = MagicMock()
|
|
321
|
+
mock_sf.get_framework_config.return_value = {
|
|
322
|
+
"packages": [
|
|
323
|
+
{"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
|
|
324
|
+
{"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
|
|
325
|
+
{"name": "dbt_constraints", "source": "git", "url": "https://github.com/Snowflake-Labs/dbt_constraints.git", "latest_version": "0.6.0"},
|
|
326
|
+
]
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
profile = ProjectProfile(
|
|
330
|
+
profile_name="test",
|
|
331
|
+
dbt_packages=["dbt_utils", "dbt_expectations"],
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
service = DownloadService(mock_sf, tmp_path)
|
|
335
|
+
service.download_dbt_packages(tmp_path, profile=profile)
|
|
336
|
+
|
|
337
|
+
import yaml
|
|
338
|
+
result = yaml.safe_load((tmp_path / "packages.yml").read_text())
|
|
339
|
+
assert len(result["packages"]) == 2
|
|
340
|
+
urls = [p.get("git") for p in result["packages"]]
|
|
341
|
+
assert "https://github.com/dbt-labs/dbt-utils.git" in urls
|
|
342
|
+
assert "https://github.com/calogica/dbt-expectations.git" in urls
|
|
343
|
+
assert "https://github.com/Snowflake-Labs/dbt_constraints.git" not in urls
|
|
344
|
+
|
|
345
|
+
def test_download_dbt_packages_excludes_upstream(self, tmp_path):
|
|
346
|
+
"""Packages provided by upstream projects should be excluded."""
|
|
347
|
+
from datasecops_cli.services.download_service import DownloadService
|
|
348
|
+
|
|
349
|
+
mock_sf = MagicMock()
|
|
350
|
+
mock_sf.get_framework_config.return_value = {
|
|
351
|
+
"packages": [
|
|
352
|
+
{"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
|
|
353
|
+
{"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
|
|
354
|
+
]
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
upstream_profile = ProjectProfile(
|
|
358
|
+
profile_name="upstream_project",
|
|
359
|
+
dbt_packages=["dbt_utils"],
|
|
360
|
+
)
|
|
361
|
+
profile = ProjectProfile(
|
|
362
|
+
profile_name="test",
|
|
363
|
+
dbt_packages=["dbt_utils", "dbt_expectations"],
|
|
364
|
+
upstream_projects=["upstream_project"],
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
service = DownloadService(mock_sf, tmp_path)
|
|
368
|
+
service.download_dbt_packages(tmp_path, profile=profile, all_profiles=[profile, upstream_profile])
|
|
369
|
+
|
|
370
|
+
import yaml
|
|
371
|
+
result = yaml.safe_load((tmp_path / "packages.yml").read_text())
|
|
372
|
+
# dbt_utils excluded (provided by upstream), only dbt_expectations + local upstream dep
|
|
373
|
+
git_packages = [p for p in result["packages"] if "git" in p]
|
|
374
|
+
local_packages = [p for p in result["packages"] if "local" in p]
|
|
375
|
+
assert len(git_packages) == 1
|
|
376
|
+
assert git_packages[0]["git"] == "https://github.com/calogica/dbt-expectations.git"
|
|
377
|
+
assert len(local_packages) == 1
|
|
378
|
+
assert local_packages[0]["local"] == "local_packages/upstream_project"
|
|
379
|
+
|
|
380
|
+
def test_download_dbt_packages_no_profile_downloads_all(self, tmp_path):
|
|
381
|
+
"""When no profile is passed, all packages should be included."""
|
|
382
|
+
from datasecops_cli.services.download_service import DownloadService
|
|
383
|
+
|
|
384
|
+
mock_sf = MagicMock()
|
|
385
|
+
mock_sf.get_framework_config.return_value = {
|
|
386
|
+
"packages": [
|
|
387
|
+
{"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
|
|
388
|
+
{"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
|
|
389
|
+
{"name": "dbt_constraints", "source": "git", "url": "https://github.com/Snowflake-Labs/dbt_constraints.git", "latest_version": "0.6.0"},
|
|
390
|
+
]
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
service = DownloadService(mock_sf, tmp_path)
|
|
394
|
+
service.download_dbt_packages(tmp_path, profile=None)
|
|
395
|
+
|
|
396
|
+
import yaml
|
|
397
|
+
result = yaml.safe_load((tmp_path / "packages.yml").read_text())
|
|
398
|
+
assert len(result["packages"]) == 3
|
|
399
|
+
|
|
400
|
+
def test_download_dbt_packages_empty_dbt_packages_downloads_all(self, tmp_path):
|
|
401
|
+
"""When profile.dbt_packages is empty, all packages should be included."""
|
|
402
|
+
from datasecops_cli.services.download_service import DownloadService
|
|
403
|
+
|
|
404
|
+
mock_sf = MagicMock()
|
|
405
|
+
mock_sf.get_framework_config.return_value = {
|
|
406
|
+
"packages": [
|
|
407
|
+
{"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
|
|
408
|
+
{"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
|
|
409
|
+
]
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
profile = ProjectProfile(profile_name="test", dbt_packages=[])
|
|
413
|
+
|
|
414
|
+
service = DownloadService(mock_sf, tmp_path)
|
|
415
|
+
service.download_dbt_packages(tmp_path, profile=profile)
|
|
416
|
+
|
|
417
|
+
import yaml
|
|
418
|
+
result = yaml.safe_load((tmp_path / "packages.yml").read_text())
|
|
419
|
+
assert len(result["packages"]) == 2
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.4.4"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/directory_scaffolder.py
RENAMED
|
File without changes
|
|
File without changes
|
{datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/linting_service.py
RENAMED
|
File without changes
|
|
File without changes
|
{datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/upstream_service.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|