datasecops-cli 0.4.4__tar.gz → 0.4.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/CHANGELOG.md +19 -0
  2. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/PKG-INFO +1 -1
  3. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/pyproject.toml +1 -1
  4. datasecops_cli-0.4.6/src/datasecops_cli/__init__.py +1 -0
  5. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/main.py +7 -1
  6. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/menus/development.py +6 -0
  7. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/menus/downloads.py +4 -2
  8. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/bootstrap_service.py +5 -2
  9. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/dbt_project_generator.py +19 -0
  10. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/download_service.py +61 -146
  11. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/snowflake_service.py +5 -0
  12. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_main.py +118 -1
  13. datasecops_cli-0.4.4/src/datasecops_cli/__init__.py +0 -1
  14. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/.github/workflows/auto-tag.yml +0 -0
  15. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/.github/workflows/publish-cli.yml +0 -0
  16. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/.gitignore +0 -0
  17. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/DEVELOPMENT.md +0 -0
  18. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/LICENSE +0 -0
  19. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/README.md +0 -0
  20. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/docs/getting-started.md +0 -0
  21. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/docs/legacy.md +0 -0
  22. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/docs/legacy_plan_of_action.md +0 -0
  23. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/docs/mcp-server.md +0 -0
  24. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/mcp-servers.json +0 -0
  25. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/setup.ps1 +0 -0
  26. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/setup.sh +0 -0
  27. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/config.py +0 -0
  28. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/menus/__init__.py +0 -0
  29. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/menus/configuration.py +0 -0
  30. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/menus/git_operations.py +0 -0
  31. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/models/__init__.py +0 -0
  32. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/models/git_helpers.py +0 -0
  33. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/models/project_config.py +0 -0
  34. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/__init__.py +0 -0
  35. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/dbt_runner.py +0 -0
  36. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/directory_scaffolder.py +0 -0
  37. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/git_service.py +0 -0
  38. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/linting_service.py +0 -0
  39. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/skill_service.py +0 -0
  40. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/services/upstream_service.py +0 -0
  41. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/utilities/__init__.py +0 -0
  42. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/utilities/display.py +0 -0
  43. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/utilities/file_utils.py +0 -0
  44. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
  45. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_mcp/__init__.py +0 -0
  46. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_mcp/__main__.py +0 -0
  47. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_mcp/connection.py +0 -0
  48. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/src/datasecops_mcp/server.py +0 -0
  49. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/__init__.py +0 -0
  50. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_config.py +0 -0
  51. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_file_utils.py +0 -0
  52. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_models.py +0 -0
  53. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_version.py +0 -0
  54. {datasecops_cli-0.4.4 → datasecops_cli-0.4.6}/tests/test_yaml_utils.py +0 -0
@@ -2,6 +2,25 @@
2
2
 
3
3
  All notable changes to the DataSecOps CLI are documented in this file.
4
4
 
5
+ ## [0.4.6] - 2026-05-18
6
+
7
+ ### Changed
8
+
9
+ - **SQLFluff config now fetched as rendered INI from native app** — `download_sqlfluff_config()` calls the new `api.get_sqlfluff_config_file()` stored procedure which returns a ready-to-use `.sqlfluff` file, matching exactly what the native app UI displays. Removes client-side INI rendering logic.
10
+
11
+ ### Added
12
+
13
+ - **Re-download .sqlfluff option in lint menu** — new option `[6] refresh config` in the SQLFluff linting menu to re-download the `.sqlfluff` configuration from the framework at any time, even if the file already exists locally.
14
+ - **`get_sqlfluff_config_file()` method** added to `SnowflakeService` for calling the new native app procedure.
15
+
16
+ ## [0.4.5] - 2026-05-18
17
+
18
+ ### Fixed
19
+
20
+ - **Package downloads now respect profile selection** — `datasecops download packages` and the interactive downloads menu now only include dbt packages enabled on the active profile, instead of downloading all packages from the global catalog. This matches the native app's Streamlit UI behavior.
21
+ - **Upstream package exclusion** — packages already provided by upstream projects are automatically excluded from `packages.yml`, preventing duplicate dependencies when using multi-project setups.
22
+ - **`generate_packages_yml()` filtering** — the dbt project generator now also filters packages by profile, fixing the same issue in the bootstrap/init path.
23
+
5
24
  ## [0.4.4] - 2026-05-18
6
25
 
7
26
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datasecops-cli
3
- Version: 0.4.4
3
+ Version: 0.4.6
4
4
  Summary: DataSecOps Framework CLI for Snowflake Native App
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "datasecops-cli"
7
- version = "0.4.4"
7
+ version = "0.4.6"
8
8
  description = "DataSecOps Framework CLI for Snowflake Native App"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -0,0 +1 @@
1
+ __version__ = "0.4.6"
@@ -565,7 +565,10 @@ def _run_download(config: Config, items: list[str],
565
565
  if not download_service.download_pipelines(platform=platform, profile_name=config.profile_name):
566
566
  failed = True
567
567
  elif item == "packages":
568
- if not download_service.download_dbt_packages(config.dbt_project_dir):
568
+ if not download_service.download_dbt_packages(
569
+ config.dbt_project_dir, profile=config.profile,
570
+ all_profiles=config.all_profiles
571
+ ):
569
572
  failed = True
570
573
  elif item == "macros":
571
574
  if not download_service.download_macros(config.profile_name, config.dbt_project_dir):
@@ -704,6 +707,7 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
704
707
  project_settings=config.project_settings,
705
708
  profile=config.profile,
706
709
  source_control=config.source_control,
710
+ all_profiles=config.all_profiles,
707
711
  )
708
712
  dl_menu.show()
709
713
 
@@ -722,6 +726,7 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
722
726
  project_dir=config.project_dir,
723
727
  project_settings=config.project_settings,
724
728
  profile=config.profile,
729
+ all_profiles=config.all_profiles,
725
730
  )
726
731
  bootstrap.run(platform=platform, install_skills=install_skills,
727
732
  skill_targets=skill_targets, run_deps=run_deps)
@@ -784,6 +789,7 @@ def _run_bootstrap(config: Config):
784
789
  project_dir=config.project_dir,
785
790
  project_settings=config.project_settings,
786
791
  profile=config.profile,
792
+ all_profiles=config.all_profiles,
787
793
  )
788
794
  success = bootstrap.run(platform=platform, install_skills=install_skills,
789
795
  skill_targets=skill_targets, run_deps=run_deps)
@@ -176,6 +176,7 @@ class DevelopmentMenu:
176
176
  menu_option(3, "find all - Lint all SQL files")
177
177
  menu_option(4, "fix all - Fix all SQL files")
178
178
  menu_option(5, "specific - Lint specific file")
179
+ menu_option(6, "refresh config- Re-download .sqlfluff from framework")
179
180
  menu_option(0, "back - Return to development menu")
180
181
  option = get_input_number("Choose an option: ")
181
182
  if option in (1, 2, 3, 4, 5) and not self._ensure_sqlfluff_config():
@@ -195,6 +196,11 @@ class DevelopmentMenu:
195
196
  path = get_input_string("Enter SQL file path (e.g. models/staging/stg_orders.sql): ")
196
197
  if path != "0":
197
198
  self.linting.lint_file(file_path=path, fix=False)
199
+ elif option == 6:
200
+ if not self.downloads:
201
+ error_line("Download service not available")
202
+ else:
203
+ self.downloads.download_sqlfluff_config(profiles_dir=self.profiles_dir, dbt_project_dir=self.linting.project_dir)
198
204
  complete_action()
199
205
 
200
206
  def _autofix_menu(self) -> None:
@@ -19,7 +19,7 @@ class DownloadsMenu:
19
19
  def __init__(self, download_service: DownloadService, skill_service: SkillService,
20
20
  dbt_runner: DbtRunner, profile_name: str, dbt_project_dir: Path,
21
21
  project_settings: ProjectSettings = None, profile: ProjectProfile = None,
22
- source_control: SourceControl = None):
22
+ source_control: SourceControl = None, all_profiles: list[ProjectProfile] = None):
23
23
  self.downloads = download_service
24
24
  self.skills = skill_service
25
25
  self.dbt = dbt_runner
@@ -28,6 +28,7 @@ class DownloadsMenu:
28
28
  self.project_settings = project_settings
29
29
  self.profile = profile
30
30
  self.source_control = source_control
31
+ self.all_profiles = all_profiles
31
32
 
32
33
  def show(self) -> None:
33
34
  self._menu()
@@ -46,7 +47,8 @@ class DownloadsMenu:
46
47
  complete_action()
47
48
  elif option == 3:
48
49
  display_action_header("Download dbt Packages")
49
- self.downloads.download_dbt_packages(self.dbt_project_dir)
50
+ self.downloads.download_dbt_packages(self.dbt_project_dir, profile=self.profile,
51
+ all_profiles=self.all_profiles)
50
52
  if get_input_true_false("Run dbt deps now?"):
51
53
  self.dbt.deps()
52
54
  complete_action()
@@ -14,11 +14,13 @@ class BootstrapService:
14
14
  """Bootstraps a new dbt project with all framework configuration."""
15
15
 
16
16
  def __init__(self, snowflake_service: SnowflakeService, project_dir: Path,
17
- project_settings: ProjectSettings, profile: ProjectProfile):
17
+ project_settings: ProjectSettings, profile: ProjectProfile,
18
+ all_profiles: list[ProjectProfile] = None):
18
19
  self.sf = snowflake_service
19
20
  self.project_dir = project_dir
20
21
  self.project_settings = project_settings
21
22
  self.profile = profile
23
+ self.all_profiles = all_profiles or []
22
24
  self.download_service = DownloadService(snowflake_service, project_dir)
23
25
  self.skill_service = SkillService(snowflake_service)
24
26
 
@@ -92,7 +94,8 @@ class BootstrapService:
92
94
  # Step 5: Generate packages.yml
93
95
  info_line("")
94
96
  info_line("[5] Generating packages.yml...")
95
- if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile):
97
+ if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile,
98
+ all_profiles=self.all_profiles):
96
99
  steps_passed += 1
97
100
  else:
98
101
  info_line(" (skipped - no dbt packages config in native app)")
@@ -165,17 +165,36 @@ def generate_profiles_yml(
165
165
  def generate_packages_yml(
166
166
  profile: ProjectProfile,
167
167
  packages_config: dict,
168
+ all_profiles: list[ProjectProfile] = None,
168
169
  ) -> str:
169
170
  """Generate packages.yml from profile's packages and upstream projects.
170
171
 
171
172
  Args:
172
173
  profile: The project profile.
173
174
  packages_config: The DBT_PACKAGES framework config.
175
+ all_profiles: All project profiles (used for upstream package exclusion).
174
176
 
175
177
  Returns:
176
178
  The YAML content as a string.
177
179
  """
178
180
  packages = packages_config.get("packages", [])
181
+
182
+ # Filter to only packages enabled on the profile
183
+ if profile.dbt_packages:
184
+ from datasecops_cli.services.download_service import _collect_upstream_packages
185
+
186
+ enabled = set(profile.dbt_packages)
187
+
188
+ # Exclude packages already provided by upstream projects
189
+ if profile.upstream_projects and all_profiles:
190
+ profile_map = {p.profile_name: p for p in all_profiles}
191
+ upstream_pkgs = _collect_upstream_packages(
192
+ profile.upstream_projects, profile_map
193
+ )
194
+ enabled -= upstream_pkgs
195
+
196
+ packages = [p for p in packages if p.get("name") in enabled]
197
+
179
198
  pkg_list: list[dict[str, Any]] = []
180
199
 
181
200
  for pkg in packages:
@@ -10,164 +10,62 @@ from datasecops_cli.utilities.display import info_line, success_line, error_line
10
10
  from datasecops_cli.utilities.file_utils import write_file, ensure_dir
11
11
 
12
12
 
13
+ def _collect_upstream_packages(
14
+ upstream_names: list[str],
15
+ profile_map: dict[str, ProjectProfile],
16
+ visited: set[str] | None = None,
17
+ ) -> set[str]:
18
+ """Recursively collect dbt package names from all upstream projects."""
19
+ if visited is None:
20
+ visited = set()
21
+ result: set[str] = set()
22
+ for name in upstream_names:
23
+ if name in visited:
24
+ continue
25
+ visited.add(name)
26
+ upstream = profile_map.get(name)
27
+ if upstream:
28
+ result.update(upstream.dbt_packages)
29
+ result.update(
30
+ _collect_upstream_packages(
31
+ upstream.upstream_projects or [], profile_map, visited
32
+ )
33
+ )
34
+ return result
35
+
36
+
13
37
  class DownloadService:
14
38
  """Downloads configurations from the native app."""
15
39
 
16
- # Maps rule codes to their sqlfluff INI section names.
17
- RULE_SECTION_MAP: dict[str, str] = {
18
- # Aliasing
19
- "AL01": "aliasing.table", "AL02": "aliasing.column", "AL03": "aliasing.expression",
20
- "AL04": "aliasing.unique.table", "AL05": "aliasing.unused", "AL06": "aliasing.length",
21
- "AL07": "aliasing.forbid", "AL08": "aliasing.unique.column", "AL09": "aliasing.self_alias",
22
- # Ambiguous
23
- "AM01": "ambiguous.distinct", "AM02": "ambiguous.union", "AM03": "ambiguous.order_by",
24
- "AM04": "ambiguous.column_count", "AM05": "ambiguous.join",
25
- "AM06": "ambiguous.column_references", "AM07": "ambiguous.set_columns",
26
- "AM08": "ambiguous.union_type",
27
- # Capitalisation
28
- "CP01": "capitalisation.keywords", "CP02": "capitalisation.identifiers",
29
- "CP03": "capitalisation.functions", "CP04": "capitalisation.literals",
30
- "CP05": "capitalisation.types",
31
- # Convention
32
- "CV01": "convention.not_equal", "CV02": "convention.coalesce",
33
- "CV03": "convention.select_trailing_comma", "CV04": "convention.count_rows",
34
- "CV05": "convention.is_null", "CV06": "convention.terminator",
35
- "CV07": "convention.statement_brackets", "CV08": "convention.left_join",
36
- "CV09": "convention.blocked_words", "CV10": "convention.quoted_literals",
37
- "CV11": "convention.casting_style", "CV12": "convention.semicolon",
38
- # Jinja
39
- "JJ01": "jinja.padding",
40
- # Layout
41
- "LT01": "layout.spacing", "LT02": "layout.indent", "LT03": "layout.operators",
42
- "LT04": "layout.commas", "LT05": "layout.long_lines", "LT06": "layout.functions",
43
- "LT07": "layout.cte_bracket", "LT08": "layout.cte_newline",
44
- "LT09": "layout.select_targets", "LT10": "layout.select_modifiers",
45
- "LT11": "layout.set_operators", "LT12": "layout.end-of-file",
46
- "LT13": "layout.start_of_file", "LT14": "layout.one_line", "LT15": "layout.newlines",
47
- # References
48
- "RF01": "references.from", "RF02": "references.qualification",
49
- "RF03": "references.consistent", "RF04": "references.keywords",
50
- "RF05": "references.special_chars", "RF06": "references.quoting",
51
- # Structure
52
- "ST01": "structure.else_null", "ST02": "structure.simple_case",
53
- "ST03": "structure.unused_cte", "ST04": "structure.nested_case",
54
- "ST05": "structure.subquery", "ST06": "structure.column_order",
55
- "ST07": "structure.using", "ST08": "structure.distinct",
56
- "ST09": "structure.join_condition_order", "ST10": "structure.cte_definition_order",
57
- "ST11": "structure.test_query",
58
- }
59
-
60
- # Core sub-keys that map to specific INI sections.
61
- CORE_SECTION_MAP: dict[str, str] = {
62
- "sqlfluff": "sqlfluff",
63
- "dbt": "sqlfluff:templater:dbt",
64
- "jinja": "sqlfluff:templater:jinja",
65
- "templater": "sqlfluff:templater",
66
- }
67
-
68
40
  def __init__(self, snowflake_service: SnowflakeService, project_dir: Path):
69
41
  self.sf = snowflake_service
70
42
  self.project_dir = project_dir
71
-
72
- # Options to strip from specific core sections (not valid sqlfluff config keys).
73
- CORE_STRIP_OPTIONS: dict[str, set[str]] = {
74
- "dbt": {"dbt_skip_compilation_error"},
75
- }
76
-
77
- @staticmethod
78
- def _format_value(val) -> str:
79
- """Format a JSON value for sqlfluff INI output."""
80
- if isinstance(val, bool):
81
- return str(val)
82
- if isinstance(val, list):
83
- if not val:
84
- return "None"
85
- return ", ".join(str(v) for v in val)
86
- if val is None or val == "":
87
- return "None"
88
- return str(val)
89
-
90
- @staticmethod
91
- def _emit_section(lines: list[str], header: str, options: dict) -> None:
92
- """Append an INI section with its options to lines."""
93
- lines.append(f"[{header}]")
94
- for key, val in options.items():
95
- lines.append(f"{key} = {DownloadService._format_value(val)}")
96
- lines.append("")
97
43
 
98
44
  def download_sqlfluff_config(self, profiles_dir: str = None, dbt_project_dir: Path = None) -> bool:
99
45
  info_line("Downloading SQLFluff configuration...")
100
- raw = self.sf.get_framework_config("SQLFLUFF_RULES")
101
- if not raw:
46
+ content = self.sf.get_sqlfluff_config_file()
47
+ if not content:
102
48
  error_line("No SQLFluff configuration found in native app")
103
49
  return False
104
-
105
- lines: list[str] = []
106
50
 
107
- # --- Core sections ([sqlfluff], [sqlfluff:templater:dbt], etc.) ---
108
- core = raw.get("core", {})
109
- for key, section_header in self.CORE_SECTION_MAP.items():
110
- entry = core.get(key)
111
- if not isinstance(entry, dict) or not entry.get("enabled", True):
112
- continue
113
- opts = dict(entry.get("options", {}))
114
- # Strip options that are not valid sqlfluff config keys
115
- for strip_key in self.CORE_STRIP_OPTIONS.get(key, set()):
116
- opts.pop(strip_key, None)
117
- if key == "dbt":
118
- # Override profiles_dir / project_dir with local values
119
- opts["project_dir"] = "."
120
- if profiles_dir:
121
- opts["profiles_dir"] = profiles_dir
122
- else:
123
- opts.pop("profiles_dir", None)
124
- if opts:
125
- self._emit_section(lines, section_header, opts)
51
+ # If profiles_dir is specified, ensure it's set in the [sqlfluff:templater:dbt] section
52
+ if profiles_dir and "[sqlfluff:templater:dbt]" in content:
53
+ import re
54
+ # Replace existing profiles_dir line if present, otherwise insert after header
55
+ if re.search(r"^profiles_dir\s*=.*$", content, re.MULTILINE):
56
+ content = re.sub(
57
+ r"^profiles_dir\s*=.*$",
58
+ f"profiles_dir = {profiles_dir}",
59
+ content,
60
+ count=1,
61
+ flags=re.MULTILINE,
62
+ )
63
+ else:
64
+ content = content.replace(
65
+ "[sqlfluff:templater:dbt]",
66
+ f"[sqlfluff:templater:dbt]\nprofiles_dir = {profiles_dir}",
67
+ )
126
68
 
127
- # --- Indentation ---
128
- indentation = raw.get("indentation", {})
129
- for _code, entry in indentation.items():
130
- if isinstance(entry, dict) and entry.get("enabled", True):
131
- opts = entry.get("options", {})
132
- if opts:
133
- self._emit_section(lines, "sqlfluff:indentation", opts)
134
-
135
- # --- Layout sections ([sqlfluff:layout:type:<code>]) ---
136
- layout = raw.get("layout", {})
137
- for code, entry in layout.items():
138
- if not isinstance(entry, dict) or not entry.get("enabled", True):
139
- continue
140
- opts = entry.get("options", {})
141
- if opts:
142
- self._emit_section(lines, f"sqlfluff:layout:type:{code}", opts)
143
-
144
- # --- Global rules ([sqlfluff:rules]) ---
145
- rules = raw.get("rules", {})
146
- for _code, entry in rules.items():
147
- if isinstance(entry, dict) and entry.get("enabled", True):
148
- opts = entry.get("options", {})
149
- if opts:
150
- self._emit_section(lines, "sqlfluff:rules", opts)
151
-
152
- # --- Rule bundle sections (rules_aliasing, rules_capitalisation, etc.) ---
153
- for section_key, entries in raw.items():
154
- if not section_key.startswith("rules_") or not isinstance(entries, dict):
155
- continue
156
- for code, entry in entries.items():
157
- if not isinstance(entry, dict) or not entry.get("enabled", True):
158
- continue
159
- section_name = self.RULE_SECTION_MAP.get(code)
160
- if not section_name:
161
- continue
162
- opts = dict(entry.get("options", {}))
163
- # For aliasing.length, 0 means "no limit" which sqlfluff expects as None
164
- if section_name == "aliasing.length":
165
- for len_key in ("min_alias_length", "max_alias_length"):
166
- if len_key in opts and opts[len_key] == 0:
167
- opts[len_key] = None
168
- self._emit_section(lines, f"sqlfluff:rules:{section_name}", opts)
169
-
170
- content = "\n".join(lines)
171
69
  dest = (dbt_project_dir or self.project_dir) / ".sqlfluff"
172
70
  write_file(dest, content)
173
71
  success_line(f"SQLFluff config written to {dest}")
@@ -279,7 +177,8 @@ class DownloadService:
279
177
  success_line(f"Downloaded {count} script(s) to {scripts_dir}")
280
178
  return True
281
179
 
282
- def download_dbt_packages(self, dbt_project_dir: Path, profile: ProjectProfile = None) -> bool:
180
+ def download_dbt_packages(self, dbt_project_dir: Path, profile: ProjectProfile = None,
181
+ all_profiles: list[ProjectProfile] = None) -> bool:
283
182
  info_line("Downloading dbt package versions...")
284
183
  raw = self.sf.get_framework_config("DBT_PACKAGES")
285
184
  if not raw:
@@ -287,6 +186,22 @@ class DownloadService:
287
186
  return False
288
187
 
289
188
  packages = raw.get("packages", [])
189
+
190
+ # Filter to only packages enabled on the active profile
191
+ if profile and profile.dbt_packages:
192
+ enabled = set(profile.dbt_packages)
193
+
194
+ # Exclude packages already provided by upstream projects
195
+ if profile.upstream_projects and all_profiles:
196
+ profile_map = {p.profile_name: p for p in all_profiles}
197
+ upstream_pkgs = _collect_upstream_packages(
198
+ profile.upstream_projects, profile_map
199
+ )
200
+ enabled -= upstream_pkgs
201
+
202
+ packages = [p for p in packages if p.get("name") in enabled]
203
+ info_line(f" Filtered to {len(packages)} package(s) for profile '{profile.profile_name}'")
204
+
290
205
  pkg_list = []
291
206
  for pkg in packages:
292
207
  source = pkg.get("source", "git")
@@ -81,3 +81,8 @@ class SnowflakeService:
81
81
 
82
82
  def get_support_contacts(self) -> list:
83
83
  return self.call_procedure("get_support_contacts") or []
84
+
85
+ def get_sqlfluff_config_file(self) -> str:
86
+ """Get the rendered .sqlfluff config INI text from the native app."""
87
+ result = self.call_procedure("get_sqlfluff_config_file")
88
+ return result if isinstance(result, str) else ""
@@ -5,6 +5,7 @@ from pathlib import Path
5
5
 
6
6
  from datasecops_cli.main import _build_parser, _run_download, DOWNLOAD_ITEMS
7
7
  from datasecops_cli.config import Config
8
+ from datasecops_cli.models.project_config import ProjectProfile
8
9
 
9
10
 
10
11
  class TestBuildParser:
@@ -64,6 +65,11 @@ class TestRunDownload:
64
65
  config.project_dir = tmp_path
65
66
  config.dbt_project_dir = tmp_path
66
67
  config.profile_name = "test_profile"
68
+ config.profile = ProjectProfile(
69
+ profile_name="test_profile",
70
+ dbt_packages=["dbt_utils", "dbt_expectations"],
71
+ )
72
+ config.all_profiles = [config.profile]
67
73
  # source_control defaults to GitHub
68
74
  return config
69
75
 
@@ -119,7 +125,9 @@ class TestRunDownload:
119
125
  _run_download(config, ["packages"])
120
126
 
121
127
  assert exc.value.code == 0
122
- mock_ds.download_dbt_packages.assert_called_once_with(tmp_path)
128
+ mock_ds.download_dbt_packages.assert_called_once_with(
129
+ tmp_path, profile=config.profile, all_profiles=config.all_profiles
130
+ )
123
131
 
124
132
  @patch("datasecops_cli.main._connect_and_load")
125
133
  def test_download_macros(self, mock_connect, tmp_path):
@@ -300,3 +308,112 @@ class TestRunDownload:
300
308
  _run_download(config, ["install-dbt"])
301
309
 
302
310
  assert exc.value.code == 1
311
+
312
+
313
+ class TestDownloadPackageFiltering:
314
+ """Tests for dbt package filtering by profile."""
315
+
316
+ def test_download_dbt_packages_filters_by_profile(self, tmp_path):
317
+ """Only packages in profile.dbt_packages should appear in packages.yml."""
318
+ from datasecops_cli.services.download_service import DownloadService
319
+
320
+ mock_sf = MagicMock()
321
+ mock_sf.get_framework_config.return_value = {
322
+ "packages": [
323
+ {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
324
+ {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
325
+ {"name": "dbt_constraints", "source": "git", "url": "https://github.com/Snowflake-Labs/dbt_constraints.git", "latest_version": "0.6.0"},
326
+ ]
327
+ }
328
+
329
+ profile = ProjectProfile(
330
+ profile_name="test",
331
+ dbt_packages=["dbt_utils", "dbt_expectations"],
332
+ )
333
+
334
+ service = DownloadService(mock_sf, tmp_path)
335
+ service.download_dbt_packages(tmp_path, profile=profile)
336
+
337
+ import yaml
338
+ result = yaml.safe_load((tmp_path / "packages.yml").read_text())
339
+ assert len(result["packages"]) == 2
340
+ urls = [p.get("git") for p in result["packages"]]
341
+ assert "https://github.com/dbt-labs/dbt-utils.git" in urls
342
+ assert "https://github.com/calogica/dbt-expectations.git" in urls
343
+ assert "https://github.com/Snowflake-Labs/dbt_constraints.git" not in urls
344
+
345
+ def test_download_dbt_packages_excludes_upstream(self, tmp_path):
346
+ """Packages provided by upstream projects should be excluded."""
347
+ from datasecops_cli.services.download_service import DownloadService
348
+
349
+ mock_sf = MagicMock()
350
+ mock_sf.get_framework_config.return_value = {
351
+ "packages": [
352
+ {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
353
+ {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
354
+ ]
355
+ }
356
+
357
+ upstream_profile = ProjectProfile(
358
+ profile_name="upstream_project",
359
+ dbt_packages=["dbt_utils"],
360
+ )
361
+ profile = ProjectProfile(
362
+ profile_name="test",
363
+ dbt_packages=["dbt_utils", "dbt_expectations"],
364
+ upstream_projects=["upstream_project"],
365
+ )
366
+
367
+ service = DownloadService(mock_sf, tmp_path)
368
+ service.download_dbt_packages(tmp_path, profile=profile, all_profiles=[profile, upstream_profile])
369
+
370
+ import yaml
371
+ result = yaml.safe_load((tmp_path / "packages.yml").read_text())
372
+ # dbt_utils excluded (provided by upstream), only dbt_expectations + local upstream dep
373
+ git_packages = [p for p in result["packages"] if "git" in p]
374
+ local_packages = [p for p in result["packages"] if "local" in p]
375
+ assert len(git_packages) == 1
376
+ assert git_packages[0]["git"] == "https://github.com/calogica/dbt-expectations.git"
377
+ assert len(local_packages) == 1
378
+ assert local_packages[0]["local"] == "local_packages/upstream_project"
379
+
380
+ def test_download_dbt_packages_no_profile_downloads_all(self, tmp_path):
381
+ """When no profile is passed, all packages should be included."""
382
+ from datasecops_cli.services.download_service import DownloadService
383
+
384
+ mock_sf = MagicMock()
385
+ mock_sf.get_framework_config.return_value = {
386
+ "packages": [
387
+ {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
388
+ {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
389
+ {"name": "dbt_constraints", "source": "git", "url": "https://github.com/Snowflake-Labs/dbt_constraints.git", "latest_version": "0.6.0"},
390
+ ]
391
+ }
392
+
393
+ service = DownloadService(mock_sf, tmp_path)
394
+ service.download_dbt_packages(tmp_path, profile=None)
395
+
396
+ import yaml
397
+ result = yaml.safe_load((tmp_path / "packages.yml").read_text())
398
+ assert len(result["packages"]) == 3
399
+
400
+ def test_download_dbt_packages_empty_dbt_packages_downloads_all(self, tmp_path):
401
+ """When profile.dbt_packages is empty, all packages should be included."""
402
+ from datasecops_cli.services.download_service import DownloadService
403
+
404
+ mock_sf = MagicMock()
405
+ mock_sf.get_framework_config.return_value = {
406
+ "packages": [
407
+ {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
408
+ {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
409
+ ]
410
+ }
411
+
412
+ profile = ProjectProfile(profile_name="test", dbt_packages=[])
413
+
414
+ service = DownloadService(mock_sf, tmp_path)
415
+ service.download_dbt_packages(tmp_path, profile=profile)
416
+
417
+ import yaml
418
+ result = yaml.safe_load((tmp_path / "packages.yml").read_text())
419
+ assert len(result["packages"]) == 2
@@ -1 +0,0 @@
1
- __version__ = "0.4.4"
File without changes
File without changes
File without changes
File without changes