dbt-fusion-package-tools 0.0.0.post1.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. dbt_fusion_package_tools-0.0.0.post1.dev0/.gitignore +28 -0
  2. dbt_fusion_package_tools-0.0.0.post1.dev0/.python-version +1 -0
  3. dbt_fusion_package_tools-0.0.0.post1.dev0/PKG-INFO +70 -0
  4. dbt_fusion_package_tools-0.0.0.post1.dev0/README.md +55 -0
  5. dbt_fusion_package_tools-0.0.0.post1.dev0/pyproject.toml +41 -0
  6. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/__init__.py +2 -0
  7. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/check_parse_conformance.py +445 -0
  8. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/compatibility.py +51 -0
  9. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/dbt_package.py +276 -0
  10. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/dbt_package_version.py +110 -0
  11. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/exceptions.py +26 -0
  12. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/fusion_version_compatibility_output.py +12852 -0
  13. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/manual_overrides.py +100 -0
  14. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/package_example.py +2 -0
  15. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/__init__.py +0 -0
  16. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/conformance_output.py +138 -0
  17. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/get_fusion_compatible_versions.py +247 -0
  18. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/get_package_hub_files.py +204 -0
  19. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/package_hub_fusion_compatibility.py +452 -0
  20. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/packages_with_fusion_compatibility_changes.py +37 -0
  21. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/update_package_hub_json.py +193 -0
  22. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/upgrade_status.py +41 -0
  23. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/version_utils.py +523 -0
  24. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/yaml/__init__.py +0 -0
  25. dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/yaml/loader.py +49 -0
@@ -0,0 +1,28 @@
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info
8
+
9
+ # Virtual environments
10
+ .venv
11
+ .nox
12
+ venv
13
+
14
+ # IDE files
15
+ .vscode/launch.json
16
+ .vscode/settings.json
17
+
18
+ # special files
19
+ config_resources.csv
20
+ project2/*
21
+
22
+ .DS_Store
23
+
24
+ # raw package hub output (because big)
25
+ src/dbt_fusion_package_tools/scripts/output/*
26
+
27
+ # dbt logs
28
+ logs
@@ -0,0 +1,70 @@
1
+ Metadata-Version: 2.5
2
+ Name: dbt-fusion-package-tools
3
+ Version: 0.0.0.post1.dev0
4
+ Summary: Add your description here
5
+ Requires-Python: <3.14,>=3.10
6
+ Requires-Dist: click<9.0,>=8.2.0
7
+ Requires-Dist: dbt-protos>=1.0.410
8
+ Requires-Dist: mashumaro<3.15,>=3.9
9
+ Requires-Dist: pyyaml>=6.0.2; python_version >= '3.13'
10
+ Requires-Dist: pyyaml>=6.0; python_version < '3.13'
11
+ Requires-Dist: requests<3.0.0
12
+ Requires-Dist: rich>=14.0.0
13
+ Requires-Dist: typer>=0.16.0
14
+ Description-Content-Type: text/markdown
15
+
16
+ # Package Management
17
+
18
+ This directory contains code used by the `packages` option in the CLI that upgrades packages in a project to a Fusion-compatible version. The code is centered on four classes:
19
+ * DbtPackageFile: represents a file (currently packages.yml or dependencies.yml) that contains package dependencies for a project
20
+ * DbtPackage: represents a package that is installed as a dependency for the project
21
+ * DbtPackageVersion: represents a specific version of a package
22
+ * DbtPackageTextFile: contains the raw lines of text from package dependency files. This is used when upgrading packages so we can replace just the version strings within a file without affecting the rest of the file layout (such as comments).
23
+
24
+ ## How the CLI works
25
+ The `packages` command calls the `upgrade_packages` function in `main.py`. This then calls:
26
+ * `generate_package_dependencies`: extracts dependencies from project's packages.yml/dependencies.yml file and identifies installed package versions in `dbt_packages`
27
+ * Returns `DbtPackageFile` if a packages.yml/dependencies.yml file is found and specifies at least one package; otherwise, None
28
+ * `check_for_package_upgrades`: traverses the dependencies in `DbtPackageFile` and for each package, determines if the current installed versions is Fusion compatible; if not, it looks for any Fusion-compatible versions of the package
29
+ * Returns list of `PackageVersionUpgradeResult`
30
+ * Length should exactly match the number of packages in the `DbtPackageFile`'s dependencies
31
+ * `upgrade_package_versions`: takes the `PackageVersionUpgradeResult` list and if any packages need updates, it identifies the required changes in packages.yml. For a dry run, it prints out the new packages.yml; otherwise, it actually makes the changes in the file.
32
+ * Returns a single `PackageUpgradeResult`
33
+ * `print_to_console` on the `PackageUpgradeResult`
34
+
35
+ `upgrade_packages` will generate an error if:
36
+ * the path specified in `--path` does not exist or isn't a directory
37
+ * `generate_package_dependencies` can't find a packages.yml or dependencies.yml
38
+ * `generate_package_dependencies` found a packages.yml or dependencies.yml but it didn't contain any package dependencies
39
+
40
+ ## Scripts
41
+
42
+ * Used to extract info used in package upgrade CLI:
43
+ * `get_package_hub_files.py`: download package information from package hub (hub.getdbt.com) for all versions of all packages
44
+ * Output: `package_output.json`
45
+ * `get_fusion_compatible_versions.py`: loads `package_output.json` and summarizes Fusion compatibility across all versions for each package
46
+ * Output: `fusion_version_compatibility_output.json` and `fusion_version_compatibility_output.py`
47
+ * Not used as an input to the package upgrade CLI:
48
+ * `packages_with_fusion_compatibility_changes.py`: reads `fusion_version_compatibility_output.py` and generates a CSV summary of packages for analytics use
49
+ * Output: `packages.csv`
50
+
51
+ `get_package_hub_files.py` and `get_fusion_compatible_version.py` are used to pull data from the public package registry (hub.getdbt.com) and extract Fusion compatibility information from available versions. This is basically a local cache of package information to bootstrap autofix. We need to know the lower bound of Fusion-compatible versions for a package but we also know that older versions of packages will not change, so caching this locally removes a lot of repetitive network calls and text parsing. Which means faster run times and fewer failures due to network issues.
52
+
53
+ The output from these two scripts produces `fusion_version_compatibility_output.py` that contains a single constant, `FUSION_VERSION_COMPATIBILITY_OUTPUT`. This is then used in `DbtPackage`'s `merge_fusion_compatibility_output` to populate compatible versions.
54
+
55
+ ## TODO
56
+ * Private packages
57
+ * Check require_dbt_version in installed private packages
58
+ * Need a way to match the dependency in packages.yml (since it doesn't have the name which is used for public packages)
59
+ * Match the version specifier type when upgrading packages
60
+ * Currently if the package config specifies a version like ">1.0.1" and we need to upgrade to 1.0.2, it gets replaced with "1.0.2"
61
+ * Should instead replace with same format like ">1.0.2"
62
+ * Get latest versions from package hub instead of using cache
63
+ * Better handling for version in package's dbt_project.yml
64
+ * Sometimes the version number in the package's dbt_project.yml doesn't actually match the release version because package hub only checks the release tag on Github, so the installed version check will set an incorrect version
65
+ * Added logic in DbtPackageFile will override the installed version if it's less than the config's version range, but this isn't 100% reliable
66
+ * Could instead refer to the package lock file to find the exact version
67
+ * But probably not a huge problem since we are only looking for the require dbt version anyway and only look for upgrades if it's missing/incompatible
68
+ * Move package parsing logic to hubcap or package hub where appropriate
69
+ * Explicit overrides at version level
70
+ * Currently in scripts/get_fusion_compatible_versions and DbtPackageVersion.is_version_explicitly_disallowed_on_fusion, but should refine logic
@@ -0,0 +1,55 @@
1
+ # Package Management
2
+
3
+ This directory contains code used by the `packages` option in the CLI that upgrades packages in a project to a Fusion-compatible version. The code is centered on four classes:
4
+ * DbtPackageFile: represents a file (currently packages.yml or dependencies.yml) that contains package dependencies for a project
5
+ * DbtPackage: represents a package that is installed as a dependency for the project
6
+ * DbtPackageVersion: represents a specific version of a package
7
+ * DbtPackageTextFile: contains the raw lines of text from package dependency files. This is used when upgrading packages so we can replace just the version strings within a file without affecting the rest of the file layout (such as comments).
8
+
9
+ ## How the CLI works
10
+ The `packages` command calls the `upgrade_packages` function in `main.py`. This then calls:
11
+ * `generate_package_dependencies`: extracts dependencies from project's packages.yml/dependencies.yml file and identifies installed package versions in `dbt_packages`
12
+ * Returns `DbtPackageFile` if a packages.yml/dependencies.yml file is found and specifies at least one package; otherwise, None
13
+ * `check_for_package_upgrades`: traverses the dependencies in `DbtPackageFile` and for each package, determines if the current installed versions is Fusion compatible; if not, it looks for any Fusion-compatible versions of the package
14
+ * Returns list of `PackageVersionUpgradeResult`
15
+ * Length should exactly match the number of packages in the `DbtPackageFile`'s dependencies
16
+ * `upgrade_package_versions`: takes the `PackageVersionUpgradeResult` list and if any packages need updates, it identifies the required changes in packages.yml. For a dry run, it prints out the new packages.yml; otherwise, it actually makes the changes in the file.
17
+ * Returns a single `PackageUpgradeResult`
18
+ * `print_to_console` on the `PackageUpgradeResult`
19
+
20
+ `upgrade_packages` will generate an error if:
21
+ * the path specified in `--path` does not exist or isn't a directory
22
+ * `generate_package_dependencies` can't find a packages.yml or dependencies.yml
23
+ * `generate_package_dependencies` found a packages.yml or dependencies.yml but it didn't contain any package dependencies
24
+
25
+ ## Scripts
26
+
27
+ * Used to extract info used in package upgrade CLI:
28
+ * `get_package_hub_files.py`: download package information from package hub (hub.getdbt.com) for all versions of all packages
29
+ * Output: `package_output.json`
30
+ * `get_fusion_compatible_versions.py`: loads `package_output.json` and summarizes Fusion compatibility across all versions for each package
31
+ * Output: `fusion_version_compatibility_output.json` and `fusion_version_compatibility_output.py`
32
+ * Not used as an input to the package upgrade CLI:
33
+ * `packages_with_fusion_compatibility_changes.py`: reads `fusion_version_compatibility_output.py` and generates a CSV summary of packages for analytics use
34
+ * Output: `packages.csv`
35
+
36
+ `get_package_hub_files.py` and `get_fusion_compatible_version.py` are used to pull data from the public package registry (hub.getdbt.com) and extract Fusion compatibility information from available versions. This is basically a local cache of package information to bootstrap autofix. We need to know the lower bound of Fusion-compatible versions for a package but we also know that older versions of packages will not change, so caching this locally removes a lot of repetitive network calls and text parsing. Which means faster run times and fewer failures due to network issues.
37
+
38
+ The output from these two scripts produces `fusion_version_compatibility_output.py` that contains a single constant, `FUSION_VERSION_COMPATIBILITY_OUTPUT`. This is then used in `DbtPackage`'s `merge_fusion_compatibility_output` to populate compatible versions.
39
+
40
+ ## TODO
41
+ * Private packages
42
+ * Check require_dbt_version in installed private packages
43
+ * Need a way to match the dependency in packages.yml (since it doesn't have the name which is used for public packages)
44
+ * Match the version specifier type when upgrading packages
45
+ * Currently if the package config specifies a version like ">1.0.1" and we need to upgrade to 1.0.2, it gets replaced with "1.0.2"
46
+ * Should instead replace with same format like ">1.0.2"
47
+ * Get latest versions from package hub instead of using cache
48
+ * Better handling for version in package's dbt_project.yml
49
+ * Sometimes the version number in the package's dbt_project.yml doesn't actually match the release version because package hub only checks the release tag on Github, so the installed version check will set an incorrect version
50
+ * Added logic in DbtPackageFile will override the installed version if it's less than the config's version range, but this isn't 100% reliable
51
+ * Could instead refer to the package lock file to find the exact version
52
+ * But probably not a huge problem since we are only looking for the require dbt version anyway and only look for upgrades if it's missing/incompatible
53
+ * Move package parsing logic to hubcap or package hub where appropriate
54
+ * Explicit overrides at version level
55
+ * Currently in scripts/get_fusion_compatible_versions and DbtPackageVersion.is_version_explicitly_disallowed_on_fusion, but should refine logic
@@ -0,0 +1,41 @@
1
+ [project]
2
+ name = "dbt-fusion-package-tools"
3
+ description = "Add your description here"
4
+ readme = "README.md"
5
+ requires-python = ">=3.10, <3.14" # can't support 3.14 without mashumaro 3.15
6
+ dependencies = [
7
+ "mashumaro>=3.9,<3.15", # matches dbt-core
8
+ "pyyaml>=6.0.2 ; python_version >= '3.13'", # need 6.0.2 for python 3.13 support
9
+ "pyyaml>=6.0 ; python_version < '3.13'", # matches dbt-core
10
+ "dbt-protos>=1.0.410",
11
+ "requests<3.0.0", # matches dbt-common
12
+ "rich>=14.0.0",
13
+ "click>=8.2.0,<9.0", # matches dbt-core
14
+ "typer>=0.16.0",
15
+ ]
16
+
17
+ dynamic = ["version"]
18
+
19
+ [project.scripts]
20
+ dbt-fusion-package-tools = "dbt_fusion_package_tools:main"
21
+ dbt-fusion-package-schema-compat = "dbt_fusion_package_tools.check_parse_conformance:main"
22
+ package-hub-compat = "dbt_fusion_package_tools.scripts.package_hub_fusion_compatibility:app"
23
+ update-package-hub = "dbt_fusion_package_tools.scripts.update_package_hub_json:app"
24
+
25
+ [tool.uv]
26
+ package = true
27
+
28
+ [build-system]
29
+ requires = ["hatchling", "uv-dynamic-versioning>=0.7.0"]
30
+ build-backend = "hatchling.build"
31
+
32
+ [tool.hatch.version]
33
+ source = "uv-dynamic-versioning"
34
+
35
+ [tool.uv-dynamic-versioning]
36
+ vcs = "git"
37
+ style = "pep440"
38
+ # Fallback when git tags are unavailable (e.g. Dependabot running uv lock in a shallow clone)
39
+ fallback-version = "0.0.0"
40
+ # Don't include the commit ID in the version - required for TestPypi
41
+ metadata = false
@@ -0,0 +1,2 @@
1
+ def main() -> None:
2
+ print("Hello from dbt-fusion-package-tools!")
@@ -0,0 +1,445 @@
1
+ """Runs dbt parse in Fusion to identify parse erors in packages."""
2
+
3
+ import json
4
+ import os
5
+ import re
6
+ import subprocess
7
+ import tarfile
8
+ from pathlib import Path
9
+ from tempfile import TemporaryDirectory
10
+ from typing import Optional
11
+
12
+ import requests
13
+ from dbtlabs.proto.public.v1.events.fusion.invocation.invocation_pb2 import Invocation
14
+ from dbtlabs.proto.public.v1.events.fusion.log.log_pb2 import LogMessage
15
+ from google.protobuf import json_format
16
+ from rich.console import Console
17
+
18
+ from dbt_fusion_package_tools.compatibility import (
19
+ FusionConformanceResult,
20
+ FusionLogMessage,
21
+ ParseConformanceLogOutput,
22
+ )
23
+ from dbt_fusion_package_tools.dbt_package_version import DbtPackageVersion
24
+ from dbt_fusion_package_tools.exceptions import FusionBinaryNotAvailable
25
+ from dbt_fusion_package_tools.yaml.loader import safe_load
26
+
27
+ console = Console()
28
+ error_console = Console(stderr=True)
29
+
30
+ """Cleans model paths displayed in Fusion error messages.
31
+
32
+ Example:
33
+ Original: ../../../../../../private/var/folders/xf/r3bz6sxx0m57976q8nb2d1fh0000gp/T/tmp0bnpllg5/models/examples/basic_example/basic_example_d_site.yml:10:9
34
+ Cleaned: models/examples/basic_example/basic_example_d_site.yml:10:9
35
+ """
36
+ _ERROR_PATH_REGEX = re.compile(r"(?:\.\./)*")
37
+
38
+
39
+ def construct_download_url_from_latest(
40
+ latest_package_version_download_url: str, target_version_download_url: str
41
+ ) -> str:
42
+ version_download_tag: str = target_version_download_url.rsplit("/", maxsplit=1)[-1]
43
+ latest_version_url: list[str] = latest_package_version_download_url.split("/")[:-1]
44
+ latest_version_url.append(version_download_tag)
45
+ return "/".join(latest_version_url)
46
+
47
+
48
+ def download_tarball_and_run_conformance(
49
+ package_name: str,
50
+ package_id: str,
51
+ package_version_str: str,
52
+ package_version_download_url: str,
53
+ latest_package_version_download_url: Optional[str],
54
+ fusion_binary: Optional[str] = None,
55
+ ) -> Optional[FusionConformanceResult]:
56
+ with TemporaryDirectory() as tmpdir:
57
+ # download tarball from version json
58
+ tar_path: Optional[Path] = None
59
+ # track exceptions across both checks
60
+ exceptions: list[str] = []
61
+ # use explicitly provided version first
62
+ try:
63
+ # Download the tarball
64
+ response = requests.get(package_version_download_url, stream=True)
65
+ response.raise_for_status()
66
+
67
+ # Save to a temporary file
68
+ tar_path = Path(tmpdir) / "archive.tar.gz"
69
+ with open(tar_path, "wb") as f:
70
+ for chunk in response.iter_content(chunk_size=8192):
71
+ if chunk:
72
+ f.write(chunk)
73
+ except requests.exceptions.HTTPError as http_error:
74
+ exceptions.append(
75
+ f"{http_error.request.url}: {http_error.response.status_code}, {http_error.response.reason}"
76
+ )
77
+ except Exception as other_error:
78
+ exceptions.append(f"Error when downloading tarball: {other_error}")
79
+ # if that errors or doesn't exist, construct from the latest version
80
+ if not tar_path and latest_package_version_download_url:
81
+ constructed_url: str = construct_download_url_from_latest(
82
+ latest_package_version_download_url, package_version_download_url
83
+ )
84
+ try:
85
+ # Download the tarball
86
+ response = requests.get(constructed_url, stream=True)
87
+ response.raise_for_status()
88
+
89
+ # Save to a temporary file
90
+ tar_path = Path(tmpdir) / "archive.tar.gz"
91
+ with open(tar_path, "wb") as f:
92
+ for chunk in response.iter_content(chunk_size=8192):
93
+ if chunk:
94
+ f.write(chunk)
95
+ except requests.exceptions.HTTPError as http_error:
96
+ exceptions.append(
97
+ f"{http_error.request.url}: {http_error.response.status_code}, {http_error.response.reason}"
98
+ )
99
+ except Exception as other_error:
100
+ exceptions.append(f"Error when downloading tarball: {other_error}")
101
+ # if still no download, error
102
+ if not tar_path:
103
+ console.log(f"Could not download {package_name} {package_version_str}")
104
+ for exception in exceptions:
105
+ console.log(exception)
106
+ return FusionConformanceResult(version=package_version_str, download_failed=True)
107
+
108
+ # if we do have a file, extract the archive
109
+ try:
110
+ extract_dir = Path(tmpdir) / "extracted"
111
+ extract_dir.mkdir(parents=True, exist_ok=True)
112
+
113
+ with tarfile.open(tar_path, "r:gz") as tar:
114
+ for entry in tar:
115
+ if os.path.isabs(entry.name) or ".." in entry.name:
116
+ raise ValueError("Illegal tar archive entry")
117
+ tar.extract(entry, extract_dir)
118
+
119
+ # Clean up the tar file
120
+ tar_path.unlink()
121
+
122
+ # Check that only 1 directory is inside
123
+ tar_contents = os.listdir(extract_dir)
124
+ if len(tar_contents) != 1:
125
+ console.log("Error downloading tar")
126
+ extracted_package = extract_dir / tar_contents[0]
127
+ except Exception as e:
128
+ console.log(f"Error when extracting tarball: {e}")
129
+ return
130
+
131
+ # run conformance if possible
132
+ try:
133
+ console.log(f"Running parse conformance for {package_id} version {package_version_str}")
134
+ return run_conformance_for_version(
135
+ extracted_package, package_name, package_version_str, package_id, fusion_binary=fusion_binary
136
+ )
137
+ except Exception as e:
138
+ console.log(f"Error when running conformance: {e}")
139
+ return
140
+
141
+
142
+ def run_conformance_for_version(
143
+ path, package_name, tag_version, package_id, fusion_binary=None
144
+ ) -> Optional[FusionConformanceResult]:
145
+ result = FusionConformanceResult(version=tag_version, download_failed=False)
146
+ # check require dbt version
147
+ try:
148
+ dbt_project_yml = safe_load((Path(f"{path}/dbt_project.yml")).read_text()) or (
149
+ {},
150
+ {},
151
+ )
152
+ require_dbt_version_string = dbt_project_yml[1].get("require-dbt-version")
153
+ except Exception as e:
154
+ error_console.log(f"dbt_project.yml load failed for {package_id} {tag_version}: {e}")
155
+ return
156
+ # try to add profile to suppress warning about profiles
157
+ if "profile" not in dbt_project_yml[1]:
158
+ console.log("Adding profile to dbt project")
159
+ try:
160
+ with open(Path(f"{path}/dbt_project.yml"), "a") as f:
161
+ f.write("\nprofile: test_schema_compat\n")
162
+ except Exception as e:
163
+ error_console.log(f"failed when adding profile to dbt_project.yml for {package_id} {tag_version}: {e}")
164
+ new_version: DbtPackageVersion = DbtPackageVersion(
165
+ package_name,
166
+ tag_version,
167
+ package_id=package_id,
168
+ raw_require_dbt_version_range=require_dbt_version_string,
169
+ )
170
+ parse_conformance = check_fusion_schema_compatibility(
171
+ Path(path), fusion_binary=fusion_binary, show_fusion_output=True
172
+ )
173
+ result.require_dbt_version_defined = new_version.is_require_dbt_version_defined()
174
+ if result.require_dbt_version_defined:
175
+ result.require_dbt_version_compatible = new_version.is_require_dbt_version_fusion_compatible()
176
+ result.manually_verified_compatible = new_version.is_explicitly_allowed_on_fusion()
177
+ result.manually_verified_incompatible = new_version.is_explicitly_disallowed_on_fusion()
178
+ if parse_conformance:
179
+ result.parse_compatible = parse_conformance.parse_exit_code == 0
180
+ result.parse_compatibility_result = parse_conformance
181
+ return result
182
+
183
+
184
+ def check_binary_name(binary_name: str) -> bool:
185
+ try:
186
+ subprocess.run(
187
+ [
188
+ binary_name,
189
+ "--version",
190
+ ],
191
+ capture_output=True,
192
+ timeout=60,
193
+ check=True,
194
+ )
195
+ return True
196
+ # indicates that the binary name is not found
197
+ except FileNotFoundError:
198
+ return False
199
+ # indicates that an error occurred when running command
200
+ except subprocess.CalledProcessError as process_error:
201
+ error_console.log(
202
+ f"CalledProcessError: {binary_name} --version exited with return code {process_error.returncode}"
203
+ )
204
+ error_console.log(process_error.stderr)
205
+ return False
206
+ except Exception as other_error:
207
+ error_console.log(f"{other_error}: An unknown exception occured when running {binary_name} --version")
208
+ return False
209
+
210
+
211
+ def check_fusion_version(binary_name: str) -> Optional[str]:
212
+ try:
213
+ version_result = subprocess.run(
214
+ [
215
+ binary_name,
216
+ "--version",
217
+ ],
218
+ check=False,
219
+ capture_output=True,
220
+ timeout=60,
221
+ text=True,
222
+ )
223
+ if "dbt-fusion" in version_result.stdout:
224
+ return version_result.stdout.split()[-1]
225
+ except Exception as e:
226
+ error_console.log(f"{e}: An unknown error occurred when checking dbt version")
227
+ return
228
+
229
+
230
+ def find_fusion_binary(custom_name: Optional[str] = None) -> Optional[str]:
231
+ possible_binary_names: list[str] = ["dbtf", "dbt"] if custom_name is None else [custom_name]
232
+
233
+ binary_names_found: set[str] = set()
234
+ # test each name
235
+ for binary_name in possible_binary_names:
236
+ # first check if name exists at all
237
+ binary_exists = check_binary_name(binary_name)
238
+ if binary_exists:
239
+ binary_names_found.add(binary_name)
240
+
241
+ if len(binary_names_found) == 0:
242
+ error_console.log("No fusion binaries found on system path, please install first")
243
+ return None
244
+
245
+ # now check version returned by each and use first one
246
+ # don't need exception handling here because previous step already did it
247
+ for valid_binary_name in binary_names_found:
248
+ version_result = check_fusion_version(valid_binary_name)
249
+ if version_result:
250
+ return valid_binary_name
251
+
252
+ # if we got to the end, then no fusion version has been found
253
+ error_console.log(f"Could not find Fusion binary, latest version output is {version_result}")
254
+ return None
255
+
256
+
257
+ def parse_log_output(
258
+ output: str, exit_code: int, repo_path: Path = Path.cwd(), fusion_version: str = "unknown"
259
+ ) -> ParseConformanceLogOutput:
260
+ log_output = [json.loads(x) for x in output.splitlines()]
261
+ result = ParseConformanceLogOutput(parse_exit_code=exit_code, fusion_version=fusion_version)
262
+ temp_path = str(repo_path)
263
+ for line in log_output:
264
+ if line.get("event_type") == "v1.public.events.fusion.log.LogMessage":
265
+ severity_text = line.get("severity_text")
266
+ # remove the tempdir path from the log message
267
+ body_no_path = (str(line.get("body"))).replace(temp_path, "")
268
+ body_no_path = body_no_path.replace("private/", "")
269
+ body = _ERROR_PATH_REGEX.sub("", body_no_path)
270
+ log_message = LogMessage()
271
+ json_format.ParseDict(line.get("attributes"), log_message, ignore_unknown_fields=True)
272
+ fusion_log_message = FusionLogMessage(
273
+ body=body,
274
+ severity_text=str(severity_text),
275
+ error_code=log_message.code,
276
+ original_severity_text=log_message.original_severity_text,
277
+ )
278
+ if severity_text == "ERROR":
279
+ result.errors.append(fusion_log_message)
280
+ elif severity_text == "WARNING":
281
+ result.warnings.append(fusion_log_message)
282
+ elif (
283
+ line.get("record_type") == "SpanEnd"
284
+ and line.get("event_type") == "v1.public.events.fusion.invocation.Invocation"
285
+ ):
286
+ invocation = Invocation()
287
+ json_format.ParseDict(line.get("attributes"), invocation, ignore_unknown_fields=True)
288
+ result.total_errors += invocation.metrics.total_errors
289
+ result.total_warnings += invocation.metrics.total_warnings
290
+
291
+ return result
292
+
293
+
294
+ def check_fusion_schema_compatibility(
295
+ repo_path: Path = Path.cwd(),
296
+ fusion_binary: Optional[str] = None,
297
+ show_fusion_output=True,
298
+ ) -> Optional[ParseConformanceLogOutput]:
299
+ """Check if a dbt package is fusion schema compatible by running 'dbtf parse'.
300
+
301
+ Args:
302
+ fusion_binary_name: name of a valid Fusion binary
303
+ repo_path: Path to the dbt package repository
304
+
305
+ Returns:
306
+ True if fusion compatible (dbtf parse exits with code 0), False otherwise
307
+ """
308
+ # Add a test profiles.yml to the current directory
309
+ profiles_path = repo_path / Path("profiles.yml")
310
+ try:
311
+ with open(profiles_path, "a") as f:
312
+ f.write(
313
+ "\n"
314
+ "test_schema_compat:\n"
315
+ " target: dev\n"
316
+ " outputs:\n"
317
+ " dev:\n"
318
+ " type: snowflake\n"
319
+ " account: none\n"
320
+ " database: none\n"
321
+ " schema: public\n"
322
+ )
323
+
324
+ # Ensure the `_DBT_FUSION_STRICT_MODE` is set (this will ensure fusion errors on schema violations)
325
+ os.environ["_DBT_FUSION_STRICT_MODE"] = "1"
326
+
327
+ # Find correct name for Fusion binary if none provided
328
+ fusion_binary_name: Optional[str] = fusion_binary
329
+ if fusion_binary_name is None:
330
+ fusion_binary_name = find_fusion_binary()
331
+ # If still no valid name, return
332
+ if fusion_binary_name is None:
333
+ raise FusionBinaryNotAvailable()
334
+
335
+ # Get the Fusion version
336
+ fusion_version: Optional[str] = check_fusion_version(fusion_binary_name)
337
+ if fusion_version is None:
338
+ raise FusionBinaryNotAvailable()
339
+
340
+ try:
341
+ # Run dbt deps to install package dependencies
342
+ if show_fusion_output:
343
+ console.log("\n\nRunning dbt deps", style="green")
344
+ deps_result = subprocess.run(
345
+ [
346
+ fusion_binary_name,
347
+ "deps",
348
+ "--profile",
349
+ "test_schema_compat",
350
+ "--project-dir",
351
+ str(repo_path),
352
+ "--log-format",
353
+ "otel",
354
+ ],
355
+ check=False,
356
+ text=True,
357
+ capture_output=True,
358
+ timeout=60,
359
+ )
360
+ deps_output = parse_log_output(
361
+ deps_result.stdout, deps_result.returncode, repo_path, fusion_version=fusion_version
362
+ )
363
+ if deps_result.returncode != 0:
364
+ error_console.log("dbt deps returned errors")
365
+ error_console.log(deps_output)
366
+
367
+ # Now try parse
368
+ if show_fusion_output:
369
+ console.log("\n\nRunning dbt parse", style="green")
370
+ parse_result = subprocess.run(
371
+ [
372
+ fusion_binary_name,
373
+ "parse",
374
+ "--profile",
375
+ "test_schema_compat",
376
+ "--project-dir",
377
+ str(repo_path),
378
+ "--log-format",
379
+ "otel",
380
+ ],
381
+ check=False,
382
+ text=True,
383
+ capture_output=True,
384
+ timeout=60,
385
+ )
386
+ parse_output = parse_log_output(
387
+ parse_result.stdout, parse_result.returncode, repo_path, fusion_version=fusion_version
388
+ )
389
+ if parse_result.returncode != 0:
390
+ error_console.log("dbt parse returned errors")
391
+ error_console.log(parse_output)
392
+ if len(deps_output.errors) > 0:
393
+ parse_output.errors.extend(deps_output.errors)
394
+ except Exception as e:
395
+ error_console.log(f"{e}: An unknown error occurred when running dbt parse")
396
+ return
397
+
398
+ # Return True if exit code is 0 (success)
399
+ is_compatible = parse_result.returncode == 0
400
+
401
+ if show_fusion_output:
402
+ if is_compatible:
403
+ console.log(f"Package at {repo_path} is fusion schema compatible")
404
+ else:
405
+ console.log(f"Package at {repo_path} is not fusion schema compatible")
406
+
407
+ # Clean up deps
408
+ if show_fusion_output:
409
+ console.log("\n\nRunning dbt clean", style="green")
410
+ subprocess.run(
411
+ [
412
+ fusion_binary_name,
413
+ "clean",
414
+ "--profile",
415
+ "test_schema_compat",
416
+ "--project-dir",
417
+ str(repo_path),
418
+ "--log-format",
419
+ "otel",
420
+ ],
421
+ check=False,
422
+ timeout=60,
423
+ text=True,
424
+ capture_output=True,
425
+ )
426
+ # Remove the test profile
427
+ os.remove(profiles_path)
428
+
429
+ return parse_output
430
+
431
+ except Exception as e:
432
+ error_console.log(f"Error checking fusion compatibility for {repo_path}: {e!s}")
433
+ try:
434
+ os.remove(profiles_path)
435
+ except Exception:
436
+ pass
437
+ return
438
+
439
+
440
+ def main():
441
+ check_fusion_schema_compatibility(repo_path=Path.cwd())
442
+
443
+
444
+ if __name__ == "__main__":
445
+ main()