dbt-fusion-package-tools 0.0.0.post1.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_fusion_package_tools-0.0.0.post1.dev0/.gitignore +28 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/.python-version +1 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/PKG-INFO +70 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/README.md +55 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/pyproject.toml +41 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/__init__.py +2 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/check_parse_conformance.py +445 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/compatibility.py +51 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/dbt_package.py +276 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/dbt_package_version.py +110 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/exceptions.py +26 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/fusion_version_compatibility_output.py +12852 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/manual_overrides.py +100 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/package_example.py +2 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/__init__.py +0 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/conformance_output.py +138 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/get_fusion_compatible_versions.py +247 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/get_package_hub_files.py +204 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/package_hub_fusion_compatibility.py +452 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/packages_with_fusion_compatibility_changes.py +37 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/scripts/update_package_hub_json.py +193 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/upgrade_status.py +41 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/version_utils.py +523 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/yaml/__init__.py +0 -0
- dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/yaml/loader.py +49 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Python-generated files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[oc]
|
|
4
|
+
build/
|
|
5
|
+
dist/
|
|
6
|
+
wheels/
|
|
7
|
+
*.egg-info
|
|
8
|
+
|
|
9
|
+
# Virtual environments
|
|
10
|
+
.venv
|
|
11
|
+
.nox
|
|
12
|
+
venv
|
|
13
|
+
|
|
14
|
+
# IDE files
|
|
15
|
+
.vscode/launch.json
|
|
16
|
+
.vscode/settings.json
|
|
17
|
+
|
|
18
|
+
# special files
|
|
19
|
+
config_resources.csv
|
|
20
|
+
project2/*
|
|
21
|
+
|
|
22
|
+
.DS_Store
|
|
23
|
+
|
|
24
|
+
# raw package hub output (because big)
|
|
25
|
+
src/dbt_fusion_package_tools/scripts/output/*
|
|
26
|
+
|
|
27
|
+
# dbt logs
|
|
28
|
+
logs
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.10
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dbt-fusion-package-tools
|
|
3
|
+
Version: 0.0.0.post1.dev0
|
|
4
|
+
Summary: Add your description here
|
|
5
|
+
Requires-Python: <3.14,>=3.10
|
|
6
|
+
Requires-Dist: click<9.0,>=8.2.0
|
|
7
|
+
Requires-Dist: dbt-protos>=1.0.410
|
|
8
|
+
Requires-Dist: mashumaro<3.15,>=3.9
|
|
9
|
+
Requires-Dist: pyyaml>=6.0.2; python_version >= '3.13'
|
|
10
|
+
Requires-Dist: pyyaml>=6.0; python_version < '3.13'
|
|
11
|
+
Requires-Dist: requests<3.0.0
|
|
12
|
+
Requires-Dist: rich>=14.0.0
|
|
13
|
+
Requires-Dist: typer>=0.16.0
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# Package Management
|
|
17
|
+
|
|
18
|
+
This directory contains code used by the `packages` option in the CLI that upgrades packages in a project to a Fusion-compatible version. The code is centered on four classes:
|
|
19
|
+
* DbtPackageFile: represents a file (currently packages.yml or dependencies.yml) that contains package dependencies for a project
|
|
20
|
+
* DbtPackage: represents a package that is installed as a dependency for the project
|
|
21
|
+
* DbtPackageVersion: represents a specific version of a package
|
|
22
|
+
* DbtPackageTextFile: contains the raw lines of text from package dependency files. This is used when upgrading packages so we can replace just the version strings within a file without affecting the rest of the file layout (such as comments).
|
|
23
|
+
|
|
24
|
+
## How the CLI works
|
|
25
|
+
The `packages` command calls the `upgrade_packages` function in `main.py`. This then calls:
|
|
26
|
+
* `generate_package_dependencies`: extracts dependencies from project's packages.yml/dependencies.yml file and identifies installed package versions in `dbt_packages`
|
|
27
|
+
* Returns `DbtPackageFile` if a packages.yml/dependencies.yml file is found and specifies at least one package; otherwise, None
|
|
28
|
+
* `check_for_package_upgrades`: traverses the dependencies in `DbtPackageFile` and for each package, determines if the current installed versions is Fusion compatible; if not, it looks for any Fusion-compatible versions of the package
|
|
29
|
+
* Returns list of `PackageVersionUpgradeResult`
|
|
30
|
+
* Length should exactly match the number of packages in the `DbtPackageFile`'s dependencies
|
|
31
|
+
* `upgrade_package_versions`: takes the `PackageVersionUpgradeResult` list and if any packages need updates, it identifies the required changes in packages.yml. For a dry run, it prints out the new packages.yml; otherwise, it actually makes the changes in the file.
|
|
32
|
+
* Returns a single `PackageUpgradeResult`
|
|
33
|
+
* `print_to_console` on the `PackageUpgradeResult`
|
|
34
|
+
|
|
35
|
+
`upgrade_packages` will generate an error if:
|
|
36
|
+
* the path specified in `--path` does not exist or isn't a directory
|
|
37
|
+
* `generate_package_dependencies` can't find a packages.yml or dependencies.yml
|
|
38
|
+
* `generate_package_dependencies` found a packages.yml or dependencies.yml but it didn't contain any package dependencies
|
|
39
|
+
|
|
40
|
+
## Scripts
|
|
41
|
+
|
|
42
|
+
* Used to extract info used in package upgrade CLI:
|
|
43
|
+
* `get_package_hub_files.py`: download package information from package hub (hub.getdbt.com) for all versions of all packages
|
|
44
|
+
* Output: `package_output.json`
|
|
45
|
+
* `get_fusion_compatible_versions.py`: loads `package_output.json` and summarizes Fusion compatibility across all versions for each package
|
|
46
|
+
* Output: `fusion_version_compatibility_output.json` and `fusion_version_compatibility_output.py`
|
|
47
|
+
* Not used as an input to the package upgrade CLI:
|
|
48
|
+
* `packages_with_fusion_compatibility_changes.py`: reads `fusion_version_compatibility_output.py` and generates a CSV summary of packages for analytics use
|
|
49
|
+
* Output: `packages.csv`
|
|
50
|
+
|
|
51
|
+
`get_package_hub_files.py` and `get_fusion_compatible_version.py` are used to pull data from the public package registry (hub.getdbt.com) and extract Fusion compatibility information from available versions. This is basically a local cache of package information to bootstrap autofix. We need to know the lower bound of Fusion-compatible versions for a package but we also know that older versions of packages will not change, so caching this locally removes a lot of repetitive network calls and text parsing. Which means faster run times and fewer failures due to network issues.
|
|
52
|
+
|
|
53
|
+
The output from these two scripts produces `fusion_version_compatibility_output.py` that contains a single constant, `FUSION_VERSION_COMPATIBILITY_OUTPUT`. This is then used in `DbtPackage`'s `merge_fusion_compatibility_output` to populate compatible versions.
|
|
54
|
+
|
|
55
|
+
## TODO
|
|
56
|
+
* Private packages
|
|
57
|
+
* Check require_dbt_version in installed private packages
|
|
58
|
+
* Need a way to match the dependency in packages.yml (since it doesn't have the name which is used for public packages)
|
|
59
|
+
* Match the version specifier type when upgrading packages
|
|
60
|
+
* Currently if the package config specifies a version like ">1.0.1" and we need to upgrade to 1.0.2, it gets replaced with "1.0.2"
|
|
61
|
+
* Should instead replace with same format like ">1.0.2"
|
|
62
|
+
* Get latest versions from package hub instead of using cache
|
|
63
|
+
* Better handling for version in package's dbt_project.yml
|
|
64
|
+
* Sometimes the version number in the package's dbt_project.yml doesn't actually match the release version because package hub only checks the release tag on Github, so the installed version check will set an incorrect version
|
|
65
|
+
* Added logic in DbtPackageFile will override the installed version if it's less than the config's version range, but this isn't 100% reliable
|
|
66
|
+
* Could instead refer to the package lock file to find the exact version
|
|
67
|
+
* But probably not a huge problem since we are only looking for the require dbt version anyway and only look for upgrades if it's missing/incompatible
|
|
68
|
+
* Move package parsing logic to hubcap or package hub where appropriate
|
|
69
|
+
* Explicit overrides at version level
|
|
70
|
+
* Currently in scripts/get_fusion_compatible_versions and DbtPackageVersion.is_version_explicitly_disallowed_on_fusion, but should refine logic
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# Package Management
|
|
2
|
+
|
|
3
|
+
This directory contains code used by the `packages` option in the CLI that upgrades packages in a project to a Fusion-compatible version. The code is centered on four classes:
|
|
4
|
+
* DbtPackageFile: represents a file (currently packages.yml or dependencies.yml) that contains package dependencies for a project
|
|
5
|
+
* DbtPackage: represents a package that is installed as a dependency for the project
|
|
6
|
+
* DbtPackageVersion: represents a specific version of a package
|
|
7
|
+
* DbtPackageTextFile: contains the raw lines of text from package dependency files. This is used when upgrading packages so we can replace just the version strings within a file without affecting the rest of the file layout (such as comments).
|
|
8
|
+
|
|
9
|
+
## How the CLI works
|
|
10
|
+
The `packages` command calls the `upgrade_packages` function in `main.py`. This then calls:
|
|
11
|
+
* `generate_package_dependencies`: extracts dependencies from project's packages.yml/dependencies.yml file and identifies installed package versions in `dbt_packages`
|
|
12
|
+
* Returns `DbtPackageFile` if a packages.yml/dependencies.yml file is found and specifies at least one package; otherwise, None
|
|
13
|
+
* `check_for_package_upgrades`: traverses the dependencies in `DbtPackageFile` and for each package, determines if the current installed versions is Fusion compatible; if not, it looks for any Fusion-compatible versions of the package
|
|
14
|
+
* Returns list of `PackageVersionUpgradeResult`
|
|
15
|
+
* Length should exactly match the number of packages in the `DbtPackageFile`'s dependencies
|
|
16
|
+
* `upgrade_package_versions`: takes the `PackageVersionUpgradeResult` list and if any packages need updates, it identifies the required changes in packages.yml. For a dry run, it prints out the new packages.yml; otherwise, it actually makes the changes in the file.
|
|
17
|
+
* Returns a single `PackageUpgradeResult`
|
|
18
|
+
* `print_to_console` on the `PackageUpgradeResult`
|
|
19
|
+
|
|
20
|
+
`upgrade_packages` will generate an error if:
|
|
21
|
+
* the path specified in `--path` does not exist or isn't a directory
|
|
22
|
+
* `generate_package_dependencies` can't find a packages.yml or dependencies.yml
|
|
23
|
+
* `generate_package_dependencies` found a packages.yml or dependencies.yml but it didn't contain any package dependencies
|
|
24
|
+
|
|
25
|
+
## Scripts
|
|
26
|
+
|
|
27
|
+
* Used to extract info used in package upgrade CLI:
|
|
28
|
+
* `get_package_hub_files.py`: download package information from package hub (hub.getdbt.com) for all versions of all packages
|
|
29
|
+
* Output: `package_output.json`
|
|
30
|
+
* `get_fusion_compatible_versions.py`: loads `package_output.json` and summarizes Fusion compatibility across all versions for each package
|
|
31
|
+
* Output: `fusion_version_compatibility_output.json` and `fusion_version_compatibility_output.py`
|
|
32
|
+
* Not used as an input to the package upgrade CLI:
|
|
33
|
+
* `packages_with_fusion_compatibility_changes.py`: reads `fusion_version_compatibility_output.py` and generates a CSV summary of packages for analytics use
|
|
34
|
+
* Output: `packages.csv`
|
|
35
|
+
|
|
36
|
+
`get_package_hub_files.py` and `get_fusion_compatible_version.py` are used to pull data from the public package registry (hub.getdbt.com) and extract Fusion compatibility information from available versions. This is basically a local cache of package information to bootstrap autofix. We need to know the lower bound of Fusion-compatible versions for a package but we also know that older versions of packages will not change, so caching this locally removes a lot of repetitive network calls and text parsing. Which means faster run times and fewer failures due to network issues.
|
|
37
|
+
|
|
38
|
+
The output from these two scripts produces `fusion_version_compatibility_output.py` that contains a single constant, `FUSION_VERSION_COMPATIBILITY_OUTPUT`. This is then used in `DbtPackage`'s `merge_fusion_compatibility_output` to populate compatible versions.
|
|
39
|
+
|
|
40
|
+
## TODO
|
|
41
|
+
* Private packages
|
|
42
|
+
* Check require_dbt_version in installed private packages
|
|
43
|
+
* Need a way to match the dependency in packages.yml (since it doesn't have the name which is used for public packages)
|
|
44
|
+
* Match the version specifier type when upgrading packages
|
|
45
|
+
* Currently if the package config specifies a version like ">1.0.1" and we need to upgrade to 1.0.2, it gets replaced with "1.0.2"
|
|
46
|
+
* Should instead replace with same format like ">1.0.2"
|
|
47
|
+
* Get latest versions from package hub instead of using cache
|
|
48
|
+
* Better handling for version in package's dbt_project.yml
|
|
49
|
+
* Sometimes the version number in the package's dbt_project.yml doesn't actually match the release version because package hub only checks the release tag on Github, so the installed version check will set an incorrect version
|
|
50
|
+
* Added logic in DbtPackageFile will override the installed version if it's less than the config's version range, but this isn't 100% reliable
|
|
51
|
+
* Could instead refer to the package lock file to find the exact version
|
|
52
|
+
* But probably not a huge problem since we are only looking for the require dbt version anyway and only look for upgrades if it's missing/incompatible
|
|
53
|
+
* Move package parsing logic to hubcap or package hub where appropriate
|
|
54
|
+
* Explicit overrides at version level
|
|
55
|
+
* Currently in scripts/get_fusion_compatible_versions and DbtPackageVersion.is_version_explicitly_disallowed_on_fusion, but should refine logic
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "dbt-fusion-package-tools"
|
|
3
|
+
description = "Add your description here"
|
|
4
|
+
readme = "README.md"
|
|
5
|
+
requires-python = ">=3.10, <3.14" # can't support 3.14 without mashumaro 3.15
|
|
6
|
+
dependencies = [
|
|
7
|
+
"mashumaro>=3.9,<3.15", # matches dbt-core
|
|
8
|
+
"pyyaml>=6.0.2 ; python_version >= '3.13'", # need 6.0.2 for python 3.13 support
|
|
9
|
+
"pyyaml>=6.0 ; python_version < '3.13'", # matches dbt-core
|
|
10
|
+
"dbt-protos>=1.0.410",
|
|
11
|
+
"requests<3.0.0", # matches dbt-common
|
|
12
|
+
"rich>=14.0.0",
|
|
13
|
+
"click>=8.2.0,<9.0", # matches dbt-core
|
|
14
|
+
"typer>=0.16.0",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
dynamic = ["version"]
|
|
18
|
+
|
|
19
|
+
[project.scripts]
|
|
20
|
+
dbt-fusion-package-tools = "dbt_fusion_package_tools:main"
|
|
21
|
+
dbt-fusion-package-schema-compat = "dbt_fusion_package_tools.check_parse_conformance:main"
|
|
22
|
+
package-hub-compat = "dbt_fusion_package_tools.scripts.package_hub_fusion_compatibility:app"
|
|
23
|
+
update-package-hub = "dbt_fusion_package_tools.scripts.update_package_hub_json:app"
|
|
24
|
+
|
|
25
|
+
[tool.uv]
|
|
26
|
+
package = true
|
|
27
|
+
|
|
28
|
+
[build-system]
|
|
29
|
+
requires = ["hatchling", "uv-dynamic-versioning>=0.7.0"]
|
|
30
|
+
build-backend = "hatchling.build"
|
|
31
|
+
|
|
32
|
+
[tool.hatch.version]
|
|
33
|
+
source = "uv-dynamic-versioning"
|
|
34
|
+
|
|
35
|
+
[tool.uv-dynamic-versioning]
|
|
36
|
+
vcs = "git"
|
|
37
|
+
style = "pep440"
|
|
38
|
+
# Fallback when git tags are unavailable (e.g. Dependabot running uv lock in a shallow clone)
|
|
39
|
+
fallback-version = "0.0.0"
|
|
40
|
+
# Don't include the commit ID in the version - required for TestPypi
|
|
41
|
+
metadata = false
|
dbt_fusion_package_tools-0.0.0.post1.dev0/src/dbt_fusion_package_tools/check_parse_conformance.py
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
"""Runs dbt parse in Fusion to identify parse erors in packages."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import subprocess
|
|
7
|
+
import tarfile
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from tempfile import TemporaryDirectory
|
|
10
|
+
from typing import Optional
|
|
11
|
+
|
|
12
|
+
import requests
|
|
13
|
+
from dbtlabs.proto.public.v1.events.fusion.invocation.invocation_pb2 import Invocation
|
|
14
|
+
from dbtlabs.proto.public.v1.events.fusion.log.log_pb2 import LogMessage
|
|
15
|
+
from google.protobuf import json_format
|
|
16
|
+
from rich.console import Console
|
|
17
|
+
|
|
18
|
+
from dbt_fusion_package_tools.compatibility import (
|
|
19
|
+
FusionConformanceResult,
|
|
20
|
+
FusionLogMessage,
|
|
21
|
+
ParseConformanceLogOutput,
|
|
22
|
+
)
|
|
23
|
+
from dbt_fusion_package_tools.dbt_package_version import DbtPackageVersion
|
|
24
|
+
from dbt_fusion_package_tools.exceptions import FusionBinaryNotAvailable
|
|
25
|
+
from dbt_fusion_package_tools.yaml.loader import safe_load
|
|
26
|
+
|
|
27
|
+
console = Console()
|
|
28
|
+
error_console = Console(stderr=True)
|
|
29
|
+
|
|
30
|
+
"""Cleans model paths displayed in Fusion error messages.
|
|
31
|
+
|
|
32
|
+
Example:
|
|
33
|
+
Original: ../../../../../../private/var/folders/xf/r3bz6sxx0m57976q8nb2d1fh0000gp/T/tmp0bnpllg5/models/examples/basic_example/basic_example_d_site.yml:10:9
|
|
34
|
+
Cleaned: models/examples/basic_example/basic_example_d_site.yml:10:9
|
|
35
|
+
"""
|
|
36
|
+
_ERROR_PATH_REGEX = re.compile(r"(?:\.\./)*")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def construct_download_url_from_latest(
|
|
40
|
+
latest_package_version_download_url: str, target_version_download_url: str
|
|
41
|
+
) -> str:
|
|
42
|
+
version_download_tag: str = target_version_download_url.rsplit("/", maxsplit=1)[-1]
|
|
43
|
+
latest_version_url: list[str] = latest_package_version_download_url.split("/")[:-1]
|
|
44
|
+
latest_version_url.append(version_download_tag)
|
|
45
|
+
return "/".join(latest_version_url)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def download_tarball_and_run_conformance(
|
|
49
|
+
package_name: str,
|
|
50
|
+
package_id: str,
|
|
51
|
+
package_version_str: str,
|
|
52
|
+
package_version_download_url: str,
|
|
53
|
+
latest_package_version_download_url: Optional[str],
|
|
54
|
+
fusion_binary: Optional[str] = None,
|
|
55
|
+
) -> Optional[FusionConformanceResult]:
|
|
56
|
+
with TemporaryDirectory() as tmpdir:
|
|
57
|
+
# download tarball from version json
|
|
58
|
+
tar_path: Optional[Path] = None
|
|
59
|
+
# track exceptions across both checks
|
|
60
|
+
exceptions: list[str] = []
|
|
61
|
+
# use explicitly provided version first
|
|
62
|
+
try:
|
|
63
|
+
# Download the tarball
|
|
64
|
+
response = requests.get(package_version_download_url, stream=True)
|
|
65
|
+
response.raise_for_status()
|
|
66
|
+
|
|
67
|
+
# Save to a temporary file
|
|
68
|
+
tar_path = Path(tmpdir) / "archive.tar.gz"
|
|
69
|
+
with open(tar_path, "wb") as f:
|
|
70
|
+
for chunk in response.iter_content(chunk_size=8192):
|
|
71
|
+
if chunk:
|
|
72
|
+
f.write(chunk)
|
|
73
|
+
except requests.exceptions.HTTPError as http_error:
|
|
74
|
+
exceptions.append(
|
|
75
|
+
f"{http_error.request.url}: {http_error.response.status_code}, {http_error.response.reason}"
|
|
76
|
+
)
|
|
77
|
+
except Exception as other_error:
|
|
78
|
+
exceptions.append(f"Error when downloading tarball: {other_error}")
|
|
79
|
+
# if that errors or doesn't exist, construct from the latest version
|
|
80
|
+
if not tar_path and latest_package_version_download_url:
|
|
81
|
+
constructed_url: str = construct_download_url_from_latest(
|
|
82
|
+
latest_package_version_download_url, package_version_download_url
|
|
83
|
+
)
|
|
84
|
+
try:
|
|
85
|
+
# Download the tarball
|
|
86
|
+
response = requests.get(constructed_url, stream=True)
|
|
87
|
+
response.raise_for_status()
|
|
88
|
+
|
|
89
|
+
# Save to a temporary file
|
|
90
|
+
tar_path = Path(tmpdir) / "archive.tar.gz"
|
|
91
|
+
with open(tar_path, "wb") as f:
|
|
92
|
+
for chunk in response.iter_content(chunk_size=8192):
|
|
93
|
+
if chunk:
|
|
94
|
+
f.write(chunk)
|
|
95
|
+
except requests.exceptions.HTTPError as http_error:
|
|
96
|
+
exceptions.append(
|
|
97
|
+
f"{http_error.request.url}: {http_error.response.status_code}, {http_error.response.reason}"
|
|
98
|
+
)
|
|
99
|
+
except Exception as other_error:
|
|
100
|
+
exceptions.append(f"Error when downloading tarball: {other_error}")
|
|
101
|
+
# if still no download, error
|
|
102
|
+
if not tar_path:
|
|
103
|
+
console.log(f"Could not download {package_name} {package_version_str}")
|
|
104
|
+
for exception in exceptions:
|
|
105
|
+
console.log(exception)
|
|
106
|
+
return FusionConformanceResult(version=package_version_str, download_failed=True)
|
|
107
|
+
|
|
108
|
+
# if we do have a file, extract the archive
|
|
109
|
+
try:
|
|
110
|
+
extract_dir = Path(tmpdir) / "extracted"
|
|
111
|
+
extract_dir.mkdir(parents=True, exist_ok=True)
|
|
112
|
+
|
|
113
|
+
with tarfile.open(tar_path, "r:gz") as tar:
|
|
114
|
+
for entry in tar:
|
|
115
|
+
if os.path.isabs(entry.name) or ".." in entry.name:
|
|
116
|
+
raise ValueError("Illegal tar archive entry")
|
|
117
|
+
tar.extract(entry, extract_dir)
|
|
118
|
+
|
|
119
|
+
# Clean up the tar file
|
|
120
|
+
tar_path.unlink()
|
|
121
|
+
|
|
122
|
+
# Check that only 1 directory is inside
|
|
123
|
+
tar_contents = os.listdir(extract_dir)
|
|
124
|
+
if len(tar_contents) != 1:
|
|
125
|
+
console.log("Error downloading tar")
|
|
126
|
+
extracted_package = extract_dir / tar_contents[0]
|
|
127
|
+
except Exception as e:
|
|
128
|
+
console.log(f"Error when extracting tarball: {e}")
|
|
129
|
+
return
|
|
130
|
+
|
|
131
|
+
# run conformance if possible
|
|
132
|
+
try:
|
|
133
|
+
console.log(f"Running parse conformance for {package_id} version {package_version_str}")
|
|
134
|
+
return run_conformance_for_version(
|
|
135
|
+
extracted_package, package_name, package_version_str, package_id, fusion_binary=fusion_binary
|
|
136
|
+
)
|
|
137
|
+
except Exception as e:
|
|
138
|
+
console.log(f"Error when running conformance: {e}")
|
|
139
|
+
return
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def run_conformance_for_version(
|
|
143
|
+
path, package_name, tag_version, package_id, fusion_binary=None
|
|
144
|
+
) -> Optional[FusionConformanceResult]:
|
|
145
|
+
result = FusionConformanceResult(version=tag_version, download_failed=False)
|
|
146
|
+
# check require dbt version
|
|
147
|
+
try:
|
|
148
|
+
dbt_project_yml = safe_load((Path(f"{path}/dbt_project.yml")).read_text()) or (
|
|
149
|
+
{},
|
|
150
|
+
{},
|
|
151
|
+
)
|
|
152
|
+
require_dbt_version_string = dbt_project_yml[1].get("require-dbt-version")
|
|
153
|
+
except Exception as e:
|
|
154
|
+
error_console.log(f"dbt_project.yml load failed for {package_id} {tag_version}: {e}")
|
|
155
|
+
return
|
|
156
|
+
# try to add profile to suppress warning about profiles
|
|
157
|
+
if "profile" not in dbt_project_yml[1]:
|
|
158
|
+
console.log("Adding profile to dbt project")
|
|
159
|
+
try:
|
|
160
|
+
with open(Path(f"{path}/dbt_project.yml"), "a") as f:
|
|
161
|
+
f.write("\nprofile: test_schema_compat\n")
|
|
162
|
+
except Exception as e:
|
|
163
|
+
error_console.log(f"failed when adding profile to dbt_project.yml for {package_id} {tag_version}: {e}")
|
|
164
|
+
new_version: DbtPackageVersion = DbtPackageVersion(
|
|
165
|
+
package_name,
|
|
166
|
+
tag_version,
|
|
167
|
+
package_id=package_id,
|
|
168
|
+
raw_require_dbt_version_range=require_dbt_version_string,
|
|
169
|
+
)
|
|
170
|
+
parse_conformance = check_fusion_schema_compatibility(
|
|
171
|
+
Path(path), fusion_binary=fusion_binary, show_fusion_output=True
|
|
172
|
+
)
|
|
173
|
+
result.require_dbt_version_defined = new_version.is_require_dbt_version_defined()
|
|
174
|
+
if result.require_dbt_version_defined:
|
|
175
|
+
result.require_dbt_version_compatible = new_version.is_require_dbt_version_fusion_compatible()
|
|
176
|
+
result.manually_verified_compatible = new_version.is_explicitly_allowed_on_fusion()
|
|
177
|
+
result.manually_verified_incompatible = new_version.is_explicitly_disallowed_on_fusion()
|
|
178
|
+
if parse_conformance:
|
|
179
|
+
result.parse_compatible = parse_conformance.parse_exit_code == 0
|
|
180
|
+
result.parse_compatibility_result = parse_conformance
|
|
181
|
+
return result
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def check_binary_name(binary_name: str) -> bool:
|
|
185
|
+
try:
|
|
186
|
+
subprocess.run(
|
|
187
|
+
[
|
|
188
|
+
binary_name,
|
|
189
|
+
"--version",
|
|
190
|
+
],
|
|
191
|
+
capture_output=True,
|
|
192
|
+
timeout=60,
|
|
193
|
+
check=True,
|
|
194
|
+
)
|
|
195
|
+
return True
|
|
196
|
+
# indicates that the binary name is not found
|
|
197
|
+
except FileNotFoundError:
|
|
198
|
+
return False
|
|
199
|
+
# indicates that an error occurred when running command
|
|
200
|
+
except subprocess.CalledProcessError as process_error:
|
|
201
|
+
error_console.log(
|
|
202
|
+
f"CalledProcessError: {binary_name} --version exited with return code {process_error.returncode}"
|
|
203
|
+
)
|
|
204
|
+
error_console.log(process_error.stderr)
|
|
205
|
+
return False
|
|
206
|
+
except Exception as other_error:
|
|
207
|
+
error_console.log(f"{other_error}: An unknown exception occured when running {binary_name} --version")
|
|
208
|
+
return False
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def check_fusion_version(binary_name: str) -> Optional[str]:
|
|
212
|
+
try:
|
|
213
|
+
version_result = subprocess.run(
|
|
214
|
+
[
|
|
215
|
+
binary_name,
|
|
216
|
+
"--version",
|
|
217
|
+
],
|
|
218
|
+
check=False,
|
|
219
|
+
capture_output=True,
|
|
220
|
+
timeout=60,
|
|
221
|
+
text=True,
|
|
222
|
+
)
|
|
223
|
+
if "dbt-fusion" in version_result.stdout:
|
|
224
|
+
return version_result.stdout.split()[-1]
|
|
225
|
+
except Exception as e:
|
|
226
|
+
error_console.log(f"{e}: An unknown error occurred when checking dbt version")
|
|
227
|
+
return
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def find_fusion_binary(custom_name: Optional[str] = None) -> Optional[str]:
|
|
231
|
+
possible_binary_names: list[str] = ["dbtf", "dbt"] if custom_name is None else [custom_name]
|
|
232
|
+
|
|
233
|
+
binary_names_found: set[str] = set()
|
|
234
|
+
# test each name
|
|
235
|
+
for binary_name in possible_binary_names:
|
|
236
|
+
# first check if name exists at all
|
|
237
|
+
binary_exists = check_binary_name(binary_name)
|
|
238
|
+
if binary_exists:
|
|
239
|
+
binary_names_found.add(binary_name)
|
|
240
|
+
|
|
241
|
+
if len(binary_names_found) == 0:
|
|
242
|
+
error_console.log("No fusion binaries found on system path, please install first")
|
|
243
|
+
return None
|
|
244
|
+
|
|
245
|
+
# now check version returned by each and use first one
|
|
246
|
+
# don't need exception handling here because previous step already did it
|
|
247
|
+
for valid_binary_name in binary_names_found:
|
|
248
|
+
version_result = check_fusion_version(valid_binary_name)
|
|
249
|
+
if version_result:
|
|
250
|
+
return valid_binary_name
|
|
251
|
+
|
|
252
|
+
# if we got to the end, then no fusion version has been found
|
|
253
|
+
error_console.log(f"Could not find Fusion binary, latest version output is {version_result}")
|
|
254
|
+
return None
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def parse_log_output(
|
|
258
|
+
output: str, exit_code: int, repo_path: Path = Path.cwd(), fusion_version: str = "unknown"
|
|
259
|
+
) -> ParseConformanceLogOutput:
|
|
260
|
+
log_output = [json.loads(x) for x in output.splitlines()]
|
|
261
|
+
result = ParseConformanceLogOutput(parse_exit_code=exit_code, fusion_version=fusion_version)
|
|
262
|
+
temp_path = str(repo_path)
|
|
263
|
+
for line in log_output:
|
|
264
|
+
if line.get("event_type") == "v1.public.events.fusion.log.LogMessage":
|
|
265
|
+
severity_text = line.get("severity_text")
|
|
266
|
+
# remove the tempdir path from the log message
|
|
267
|
+
body_no_path = (str(line.get("body"))).replace(temp_path, "")
|
|
268
|
+
body_no_path = body_no_path.replace("private/", "")
|
|
269
|
+
body = _ERROR_PATH_REGEX.sub("", body_no_path)
|
|
270
|
+
log_message = LogMessage()
|
|
271
|
+
json_format.ParseDict(line.get("attributes"), log_message, ignore_unknown_fields=True)
|
|
272
|
+
fusion_log_message = FusionLogMessage(
|
|
273
|
+
body=body,
|
|
274
|
+
severity_text=str(severity_text),
|
|
275
|
+
error_code=log_message.code,
|
|
276
|
+
original_severity_text=log_message.original_severity_text,
|
|
277
|
+
)
|
|
278
|
+
if severity_text == "ERROR":
|
|
279
|
+
result.errors.append(fusion_log_message)
|
|
280
|
+
elif severity_text == "WARNING":
|
|
281
|
+
result.warnings.append(fusion_log_message)
|
|
282
|
+
elif (
|
|
283
|
+
line.get("record_type") == "SpanEnd"
|
|
284
|
+
and line.get("event_type") == "v1.public.events.fusion.invocation.Invocation"
|
|
285
|
+
):
|
|
286
|
+
invocation = Invocation()
|
|
287
|
+
json_format.ParseDict(line.get("attributes"), invocation, ignore_unknown_fields=True)
|
|
288
|
+
result.total_errors += invocation.metrics.total_errors
|
|
289
|
+
result.total_warnings += invocation.metrics.total_warnings
|
|
290
|
+
|
|
291
|
+
return result
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def check_fusion_schema_compatibility(
|
|
295
|
+
repo_path: Path = Path.cwd(),
|
|
296
|
+
fusion_binary: Optional[str] = None,
|
|
297
|
+
show_fusion_output=True,
|
|
298
|
+
) -> Optional[ParseConformanceLogOutput]:
|
|
299
|
+
"""Check if a dbt package is fusion schema compatible by running 'dbtf parse'.
|
|
300
|
+
|
|
301
|
+
Args:
|
|
302
|
+
fusion_binary_name: name of a valid Fusion binary
|
|
303
|
+
repo_path: Path to the dbt package repository
|
|
304
|
+
|
|
305
|
+
Returns:
|
|
306
|
+
True if fusion compatible (dbtf parse exits with code 0), False otherwise
|
|
307
|
+
"""
|
|
308
|
+
# Add a test profiles.yml to the current directory
|
|
309
|
+
profiles_path = repo_path / Path("profiles.yml")
|
|
310
|
+
try:
|
|
311
|
+
with open(profiles_path, "a") as f:
|
|
312
|
+
f.write(
|
|
313
|
+
"\n"
|
|
314
|
+
"test_schema_compat:\n"
|
|
315
|
+
" target: dev\n"
|
|
316
|
+
" outputs:\n"
|
|
317
|
+
" dev:\n"
|
|
318
|
+
" type: snowflake\n"
|
|
319
|
+
" account: none\n"
|
|
320
|
+
" database: none\n"
|
|
321
|
+
" schema: public\n"
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
# Ensure the `_DBT_FUSION_STRICT_MODE` is set (this will ensure fusion errors on schema violations)
|
|
325
|
+
os.environ["_DBT_FUSION_STRICT_MODE"] = "1"
|
|
326
|
+
|
|
327
|
+
# Find correct name for Fusion binary if none provided
|
|
328
|
+
fusion_binary_name: Optional[str] = fusion_binary
|
|
329
|
+
if fusion_binary_name is None:
|
|
330
|
+
fusion_binary_name = find_fusion_binary()
|
|
331
|
+
# If still no valid name, return
|
|
332
|
+
if fusion_binary_name is None:
|
|
333
|
+
raise FusionBinaryNotAvailable()
|
|
334
|
+
|
|
335
|
+
# Get the Fusion version
|
|
336
|
+
fusion_version: Optional[str] = check_fusion_version(fusion_binary_name)
|
|
337
|
+
if fusion_version is None:
|
|
338
|
+
raise FusionBinaryNotAvailable()
|
|
339
|
+
|
|
340
|
+
try:
|
|
341
|
+
# Run dbt deps to install package dependencies
|
|
342
|
+
if show_fusion_output:
|
|
343
|
+
console.log("\n\nRunning dbt deps", style="green")
|
|
344
|
+
deps_result = subprocess.run(
|
|
345
|
+
[
|
|
346
|
+
fusion_binary_name,
|
|
347
|
+
"deps",
|
|
348
|
+
"--profile",
|
|
349
|
+
"test_schema_compat",
|
|
350
|
+
"--project-dir",
|
|
351
|
+
str(repo_path),
|
|
352
|
+
"--log-format",
|
|
353
|
+
"otel",
|
|
354
|
+
],
|
|
355
|
+
check=False,
|
|
356
|
+
text=True,
|
|
357
|
+
capture_output=True,
|
|
358
|
+
timeout=60,
|
|
359
|
+
)
|
|
360
|
+
deps_output = parse_log_output(
|
|
361
|
+
deps_result.stdout, deps_result.returncode, repo_path, fusion_version=fusion_version
|
|
362
|
+
)
|
|
363
|
+
if deps_result.returncode != 0:
|
|
364
|
+
error_console.log("dbt deps returned errors")
|
|
365
|
+
error_console.log(deps_output)
|
|
366
|
+
|
|
367
|
+
# Now try parse
|
|
368
|
+
if show_fusion_output:
|
|
369
|
+
console.log("\n\nRunning dbt parse", style="green")
|
|
370
|
+
parse_result = subprocess.run(
|
|
371
|
+
[
|
|
372
|
+
fusion_binary_name,
|
|
373
|
+
"parse",
|
|
374
|
+
"--profile",
|
|
375
|
+
"test_schema_compat",
|
|
376
|
+
"--project-dir",
|
|
377
|
+
str(repo_path),
|
|
378
|
+
"--log-format",
|
|
379
|
+
"otel",
|
|
380
|
+
],
|
|
381
|
+
check=False,
|
|
382
|
+
text=True,
|
|
383
|
+
capture_output=True,
|
|
384
|
+
timeout=60,
|
|
385
|
+
)
|
|
386
|
+
parse_output = parse_log_output(
|
|
387
|
+
parse_result.stdout, parse_result.returncode, repo_path, fusion_version=fusion_version
|
|
388
|
+
)
|
|
389
|
+
if parse_result.returncode != 0:
|
|
390
|
+
error_console.log("dbt parse returned errors")
|
|
391
|
+
error_console.log(parse_output)
|
|
392
|
+
if len(deps_output.errors) > 0:
|
|
393
|
+
parse_output.errors.extend(deps_output.errors)
|
|
394
|
+
except Exception as e:
|
|
395
|
+
error_console.log(f"{e}: An unknown error occurred when running dbt parse")
|
|
396
|
+
return
|
|
397
|
+
|
|
398
|
+
# Return True if exit code is 0 (success)
|
|
399
|
+
is_compatible = parse_result.returncode == 0
|
|
400
|
+
|
|
401
|
+
if show_fusion_output:
|
|
402
|
+
if is_compatible:
|
|
403
|
+
console.log(f"Package at {repo_path} is fusion schema compatible")
|
|
404
|
+
else:
|
|
405
|
+
console.log(f"Package at {repo_path} is not fusion schema compatible")
|
|
406
|
+
|
|
407
|
+
# Clean up deps
|
|
408
|
+
if show_fusion_output:
|
|
409
|
+
console.log("\n\nRunning dbt clean", style="green")
|
|
410
|
+
subprocess.run(
|
|
411
|
+
[
|
|
412
|
+
fusion_binary_name,
|
|
413
|
+
"clean",
|
|
414
|
+
"--profile",
|
|
415
|
+
"test_schema_compat",
|
|
416
|
+
"--project-dir",
|
|
417
|
+
str(repo_path),
|
|
418
|
+
"--log-format",
|
|
419
|
+
"otel",
|
|
420
|
+
],
|
|
421
|
+
check=False,
|
|
422
|
+
timeout=60,
|
|
423
|
+
text=True,
|
|
424
|
+
capture_output=True,
|
|
425
|
+
)
|
|
426
|
+
# Remove the test profile
|
|
427
|
+
os.remove(profiles_path)
|
|
428
|
+
|
|
429
|
+
return parse_output
|
|
430
|
+
|
|
431
|
+
except Exception as e:
|
|
432
|
+
error_console.log(f"Error checking fusion compatibility for {repo_path}: {e!s}")
|
|
433
|
+
try:
|
|
434
|
+
os.remove(profiles_path)
|
|
435
|
+
except Exception:
|
|
436
|
+
pass
|
|
437
|
+
return
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def main():
|
|
441
|
+
check_fusion_schema_compatibility(repo_path=Path.cwd())
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
if __name__ == "__main__":
|
|
445
|
+
main()
|