brickie-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brickie/__init__.py +0 -0
- brickie/build/__init__.py +0 -0
- brickie/build/bundle.py +126 -0
- brickie/build/compiler.py +57 -0
- brickie/check/__init__.py +0 -0
- brickie/check/platform_objects.py +72 -0
- brickie/check/rules.py +371 -0
- brickie/cli.py +586 -0
- brickie/conditions.py +41 -0
- brickie/deploy/__init__.py +0 -0
- brickie/deploy/databricks.py +146 -0
- brickie/deploy/databricks_cli_release.py +13 -0
- brickie/deploy/deploy.py +71 -0
- brickie/deploy/pinned_cli.py +185 -0
- brickie/diagnostics.py +75 -0
- brickie/examples/kettle/brickie.platform.yaml +21 -0
- brickie/examples/kettle/data/day-1/customers/customers-day-1.json +40 -0
- brickie/examples/kettle/data/day-1/orders/orders-day-1.json +150 -0
- brickie/examples/kettle/data/day-1/product_prices/product_prices-day-1.json +6 -0
- brickie/examples/kettle/data/day-2/customers/customers-day-2.json +2 -0
- brickie/examples/kettle/data/day-2/orders/orders-day-2.json +100 -0
- brickie/examples/kettle/data/day-2/product_prices/product_prices-day-2.json +1 -0
- brickie/examples/kettle/domains/finance/models/gold/fct_revenue.sql +10 -0
- brickie/examples/kettle/domains/finance/models/gold/fct_revenue.yaml +4 -0
- brickie/examples/kettle/domains/sales/models/gold/dim_product_prices.sql +10 -0
- brickie/examples/kettle/domains/sales/models/gold/dim_product_prices.yaml +4 -0
- brickie/examples/kettle/domains/sales/models/gold/fct_orders.sql +16 -0
- brickie/examples/kettle/domains/sales/models/gold/fct_orders.yaml +4 -0
- brickie/examples/kettle/domains/sales/seeds/regions.csv +5 -0
- brickie/examples/kettle/domains/sales/seeds/regions.yaml +7 -0
- brickie/examples/kettle/domains/sales/sources/customers.yaml +14 -0
- brickie/examples/kettle/domains/sales/sources/orders.yaml +10 -0
- brickie/examples/kettle/domains/sales/sources/product_prices.yaml +5 -0
- brickie/examples/kettle/domains/shared/models/gold/dim_customers.sql +9 -0
- brickie/examples/kettle/domains/shared/models/gold/dim_customers.yaml +4 -0
- brickie/examples/kettle/domains/shared/models/gold/dim_date.sql +9 -0
- brickie/examples/kettle/domains/shared/models/gold/dim_date.yaml +4 -0
- brickie/examples/kettle/gitignore +1 -0
- brickie/kinds/__init__.py +24 -0
- brickie/kinds/base.py +75 -0
- brickie/kinds/model.py +109 -0
- brickie/kinds/platform.py +443 -0
- brickie/kinds/seed.py +279 -0
- brickie/kinds/source.py +346 -0
- brickie/load/__init__.py +0 -0
- brickie/load/config.py +155 -0
- brickie/load/project.py +203 -0
- brickie/names.py +153 -0
- brickie/output.py +49 -0
- brickie/reference.py +351 -0
- brickie/registry.py +967 -0
- brickie/scaffold.py +116 -0
- brickie/skills.py +185 -0
- brickie/sql.py +261 -0
- brickie/statements.py +25 -0
- brickie/templates/ci/github/.github/workflows/brickie-merge.yaml +24 -0
- brickie/templates/ci/github/.github/workflows/brickie-pull-request.yaml +23 -0
- brickie/templates/project/brickie.platform.yaml +17 -0
- brickie/templates/project/domains/sales/models/gold/dim_date.sql +9 -0
- brickie/templates/project/domains/sales/models/gold/dim_date.yaml +4 -0
- brickie/templates/project/domains/sales/models/gold/fct_orders.sql +2 -0
- brickie/templates/project/domains/sales/models/gold/fct_orders.yaml +4 -0
- brickie/templates/project/domains/sales/sources/orders.yaml +5 -0
- brickie/templates/project/gitignore +1 -0
- brickie/templates/skills/brickie-add-model/SKILL.md +17 -0
- brickie/templates/skills/brickie-add-source/SKILL.md +16 -0
- brickie_core-0.1.0.dist-info/METADATA +90 -0
- brickie_core-0.1.0.dist-info/RECORD +71 -0
- brickie_core-0.1.0.dist-info/WHEEL +4 -0
- brickie_core-0.1.0.dist-info/entry_points.txt +2 -0
- brickie_core-0.1.0.dist-info/licenses/LICENSE +202 -0
brickie/__init__.py
ADDED
|
File without changes
|
|
File without changes
|
brickie/build/bundle.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Writes `databricks.yaml`: the resources, plus the placeholder values for each target."""
|
|
2
|
+
|
|
3
|
+
import yaml
|
|
4
|
+
|
|
5
|
+
from brickie import names, sql
|
|
6
|
+
from brickie.load.project import Project
|
|
7
|
+
|
|
8
|
+
REFRESH_JOB = "refresh"
|
|
9
|
+
|
|
10
|
+
# One retry survives a one-off blip. An error a retry can't fix, such as a bad row in append-only
|
|
11
|
+
# bronze, then fails in about a minute instead of waiting through the default five retries.
|
|
12
|
+
UPDATE_RETRIES = {"pipelines.numUpdateRetryAttempts": "1"}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def render(project: Project) -> str:
|
|
16
|
+
platform = project.platform
|
|
17
|
+
variables = list(names.settings(project, next(iter(platform.hosts))))
|
|
18
|
+
pipelines = {}
|
|
19
|
+
for domain in platform.domains:
|
|
20
|
+
if any(item.domain == domain for item in (*project.sources, *project.seeds)):
|
|
21
|
+
pipelines[f"{domain}_ingest"] = _pipeline(project, domain, "ingest")
|
|
22
|
+
if any(model.domain == domain for model in project.models):
|
|
23
|
+
pipelines[f"{domain}_gold"] = _pipeline(project, domain, "gold")
|
|
24
|
+
upstream = _upstream(project, set(pipelines))
|
|
25
|
+
tasks = [_task(pipeline, upstream.get(pipeline, [])) for pipeline in pipelines]
|
|
26
|
+
return yaml.safe_dump(
|
|
27
|
+
{
|
|
28
|
+
"bundle": {"name": platform.name},
|
|
29
|
+
# Bundle sync skips gitignored files, and `brickie init` gitignores the package.
|
|
30
|
+
"sync": {"include": ["src/**"]},
|
|
31
|
+
"variables": {name: {"description": name} for name in variables},
|
|
32
|
+
"targets": {env: _target(project, env) for env in platform.hosts},
|
|
33
|
+
"resources": {
|
|
34
|
+
"pipelines": pipelines,
|
|
35
|
+
"jobs": {REFRESH_JOB: {"name": _name(project, "refresh"), "tasks": tasks}},
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
sort_keys=False,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _name(project: Project, *parts: str) -> str:
|
|
43
|
+
"""A pipeline or Job name, with the Environment in it, so Environments can share a workspace.
|
|
44
|
+
|
|
45
|
+
Databricks refuses a second pipeline with the same name.
|
|
46
|
+
"""
|
|
47
|
+
return "_".join((project.platform.name, "${bundle.target}", *parts))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _target(project: Project, env: str) -> dict:
|
|
51
|
+
"""One Environment's workspace and placeholder values, and its Job's schedule if it has one."""
|
|
52
|
+
target: dict = {
|
|
53
|
+
"workspace": {"host": project.platform.hosts[env]},
|
|
54
|
+
"variables": names.settings(project, env),
|
|
55
|
+
}
|
|
56
|
+
schedule = project.platform.environments[env].schedule
|
|
57
|
+
if schedule:
|
|
58
|
+
paused = schedule.paused
|
|
59
|
+
if paused is None:
|
|
60
|
+
paused = env != project.platform.production_environment
|
|
61
|
+
target["resources"] = {
|
|
62
|
+
"jobs": {
|
|
63
|
+
REFRESH_JOB: {
|
|
64
|
+
"schedule": {
|
|
65
|
+
"quartz_cron_expression": schedule.cron,
|
|
66
|
+
"timezone_id": schedule.timezone,
|
|
67
|
+
"pause_status": "PAUSED" if paused else "UNPAUSED",
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return target
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _pipeline(project: Project, domain: str, group: str) -> dict:
|
|
76
|
+
platform = project.platform
|
|
77
|
+
# Every table is written by its full name, so this is only the pipeline's default. Model
|
|
78
|
+
# Layers have no order, so the first by name keeps it the same however the list is ordered.
|
|
79
|
+
layer = platform.source_layers[0] if group == "ingest" else min(platform.model_layers)
|
|
80
|
+
keys = names.settings(project, next(iter(platform.hosts)))
|
|
81
|
+
return {
|
|
82
|
+
"name": _name(project, domain, group),
|
|
83
|
+
"catalog": f"${{var.{names.catalog_setting(domain, layer)}}}",
|
|
84
|
+
"schema": f"${{var.{names.schema_setting(domain, layer)}}}",
|
|
85
|
+
"serverless": True,
|
|
86
|
+
"channel": "CURRENT",
|
|
87
|
+
"libraries": [{"glob": {"include": f"src/{domain}/{group}/**"}}],
|
|
88
|
+
"configuration": {**{key: f"${{var.{key}}}" for key in keys}, **UPDATE_RETRIES},
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _upstream(project: Project, pipelines: set[str]) -> dict[str, list[str]]:
|
|
93
|
+
"""The pipelines each gold pipeline reads from, so its task waits only for those.
|
|
94
|
+
|
|
95
|
+
Ingest pipelines read only files, so they wait for nothing. Silver comes from its Domain's
|
|
96
|
+
ingest pipeline, and gold of another Domain, which can only be the Shared Domain's, from its
|
|
97
|
+
gold pipeline.
|
|
98
|
+
"""
|
|
99
|
+
source_layers = {layer.lower() for layer in project.platform.source_layers}
|
|
100
|
+
# Logical Names are lowercase, and pipelines are keyed by the Domain as the contract writes it.
|
|
101
|
+
domains = {domain.lower(): domain for domain in project.platform.domains}
|
|
102
|
+
upstream: dict[str, list[str]] = {}
|
|
103
|
+
for model in project.models:
|
|
104
|
+
try:
|
|
105
|
+
reads = sql.reads(model.query)
|
|
106
|
+
except sql.Unparseable:
|
|
107
|
+
continue # `brickie validate` reports it, and `brickie deploy` refuses it.
|
|
108
|
+
reader = f"{model.domain}_gold"
|
|
109
|
+
needed = upstream.setdefault(reader, [])
|
|
110
|
+
for read in sorted(reads, key=str):
|
|
111
|
+
group = "ingest" if read.layer in source_layers else "gold"
|
|
112
|
+
pipeline = f"{domains.get(read.domain, read.domain)}_{group}"
|
|
113
|
+
if pipeline in pipelines and pipeline != reader and pipeline not in needed:
|
|
114
|
+
needed.append(pipeline)
|
|
115
|
+
return upstream
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _task(pipeline: str, needs: list[str]) -> dict:
|
|
119
|
+
task: dict = {
|
|
120
|
+
"task_key": pipeline,
|
|
121
|
+
"max_retries": 0,
|
|
122
|
+
"pipeline_task": {"pipeline_id": f"${{resources.pipelines.{pipeline}.id}}"},
|
|
123
|
+
}
|
|
124
|
+
if needs:
|
|
125
|
+
task["depends_on"] = [{"task_key": needed} for needed in needs]
|
|
126
|
+
return task
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Turns a Project into generated code: one file per table group, same in every Environment."""
|
|
2
|
+
|
|
3
|
+
import shutil
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from brickie import names, sql
|
|
7
|
+
from brickie.build import bundle
|
|
8
|
+
from brickie.kinds import model, seed, source
|
|
9
|
+
from brickie.load.project import Project
|
|
10
|
+
from brickie.names import LogicalName
|
|
11
|
+
|
|
12
|
+
BUILD_FOLDER = "build"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def generated_files(project: Project) -> dict[Path, str]:
|
|
16
|
+
"""Generated code by path relative to the package, using placeholders."""
|
|
17
|
+
files: dict[Path, str] = {}
|
|
18
|
+
bronze_layer, silver_layer = project.platform.source_layers
|
|
19
|
+
for item in project.sources:
|
|
20
|
+
bronze = names.placeholders(LogicalName(item.domain, bronze_layer, item.name))
|
|
21
|
+
silver = names.placeholders(LogicalName(item.domain, silver_layer, item.name))
|
|
22
|
+
quarantine = names.placeholders(LogicalName(item.domain, silver_layer, item.quarantine))
|
|
23
|
+
path = Path("src") / item.domain / "ingest" / f"{item.name}.sql"
|
|
24
|
+
files[path] = source.generate(
|
|
25
|
+
item, names.path_placeholder(item), bronze, silver, quarantine
|
|
26
|
+
)
|
|
27
|
+
for item in project.seeds:
|
|
28
|
+
target = names.placeholders(LogicalName(item.domain, silver_layer, item.name))
|
|
29
|
+
files[Path("src") / item.domain / "ingest" / f"{item.name}.sql"] = seed.generate(
|
|
30
|
+
item, target
|
|
31
|
+
)
|
|
32
|
+
for item in project.models:
|
|
33
|
+
layer = item.layer
|
|
34
|
+
target = names.placeholders(LogicalName(item.domain, layer, item.name))
|
|
35
|
+
path = Path("src") / item.domain / "gold" / f"{item.name}.sql"
|
|
36
|
+
query = sql.escape_placeholders(item.query)
|
|
37
|
+
rewritten = sql.rewrite(query, names.placeholders)
|
|
38
|
+
files[path] = model.generate(item, target, rewritten, sql.streams(item.query))
|
|
39
|
+
return files
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def write_package(project: Project) -> Path:
|
|
43
|
+
package = project.root / BUILD_FOLDER
|
|
44
|
+
shutil.rmtree(package, ignore_errors=True)
|
|
45
|
+
package.mkdir()
|
|
46
|
+
for relative, content in generated_files(project).items():
|
|
47
|
+
target = package / relative
|
|
48
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
49
|
+
target.write_text(content)
|
|
50
|
+
(package / "databricks.yaml").write_text(bundle.render(project))
|
|
51
|
+
return package
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def print_for(project: Project, env: str) -> str:
|
|
55
|
+
"""Every generated file as runnable SQL for one Environment."""
|
|
56
|
+
code = "\n".join(generated_files(project).values())
|
|
57
|
+
return names.resolve(code, project, env)
|
|
File without changes
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Platform rules. `brickie validate --platform` runs these against one Environment's workspace.
|
|
2
|
+
|
|
3
|
+
Brickie never creates catalogs or schemas. Whoever owns the platform does, so a missing one fails
|
|
4
|
+
here, before a deploy, rather than halfway through one.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from typing import Protocol
|
|
8
|
+
|
|
9
|
+
from brickie import names
|
|
10
|
+
from brickie.diagnostics import Diagnostic, error
|
|
11
|
+
from brickie.kinds import platform
|
|
12
|
+
from brickie.load.project import Project
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class UnityCatalog(Protocol):
|
|
16
|
+
"""What the Databricks boundary reports about a workspace's catalogs and schemas."""
|
|
17
|
+
|
|
18
|
+
def catalogs(self, host: str) -> set[str]:
|
|
19
|
+
"""The catalogs the signed-in identity can see."""
|
|
20
|
+
...
|
|
21
|
+
|
|
22
|
+
def schemas(self, host: str, catalog: str) -> set[str]:
|
|
23
|
+
"""The schemas in one catalog the signed-in identity can see."""
|
|
24
|
+
...
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def check(project: Project, env: str, workspace: UnityCatalog) -> list[Diagnostic]:
|
|
28
|
+
host = project.platform.hosts[env]
|
|
29
|
+
catalogs, schemas = names.platform_objects(project.platform, [env])
|
|
30
|
+
present = _lowercase(workspace.catalogs(host))
|
|
31
|
+
diagnostics = []
|
|
32
|
+
for catalog in catalogs:
|
|
33
|
+
needed = [schema for schema in schemas if schema.split(".")[0] == catalog]
|
|
34
|
+
if catalog.lower() not in present:
|
|
35
|
+
diagnostics.append(_missing_catalog(env, host, catalog, needed))
|
|
36
|
+
continue
|
|
37
|
+
found = _lowercase(workspace.schemas(host, catalog))
|
|
38
|
+
diagnostics += [
|
|
39
|
+
_missing_schema(env, host, schema)
|
|
40
|
+
for schema in needed
|
|
41
|
+
if schema.split(".")[1].lower() not in found
|
|
42
|
+
]
|
|
43
|
+
return diagnostics
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _lowercase(found: set[str]) -> set[str]:
|
|
47
|
+
"""Unity Catalog matches names whatever their case."""
|
|
48
|
+
return {name.lower() for name in found}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _missing_catalog(env: str, host: str, catalog: str, schemas: list[str]) -> Diagnostic:
|
|
52
|
+
listed = ", ".join(f"`{schema}`" for schema in schemas)
|
|
53
|
+
return error(
|
|
54
|
+
"BRK301",
|
|
55
|
+
platform.FILE_NAME,
|
|
56
|
+
f"The catalog `{catalog}` isn't in {env}'s workspace, {host}, or the identity Brickie"
|
|
57
|
+
f" signed in as can't see it. The project's tables in {env} need it.",
|
|
58
|
+
f"Create the catalog `{catalog}` with the schemas {listed}, or grant the identity"
|
|
59
|
+
" `USE CATALOG` on it. Brickie never creates catalogs.",
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _missing_schema(env: str, host: str, schema: str) -> Diagnostic:
|
|
64
|
+
catalog, name = schema.split(".")
|
|
65
|
+
return error(
|
|
66
|
+
"BRK302",
|
|
67
|
+
platform.FILE_NAME,
|
|
68
|
+
f"The schema `{schema}` isn't in {env}'s workspace, {host}, or the identity Brickie"
|
|
69
|
+
f" signed in as can't see it. The project's tables in {env} need it.",
|
|
70
|
+
f"Create the schema `{name}` in the catalog `{catalog}`, or grant the identity"
|
|
71
|
+
" `USE SCHEMA` on it. Brickie never creates schemas.",
|
|
72
|
+
)
|
brickie/check/rules.py
ADDED
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
"""Project rules. `brickie validate` runs these with no workspace."""
|
|
2
|
+
|
|
3
|
+
from brickie import names, sql
|
|
4
|
+
from brickie.diagnostics import Diagnostic, error
|
|
5
|
+
from brickie.kinds.model import Model
|
|
6
|
+
from brickie.kinds.platform import SHARED_DOMAIN
|
|
7
|
+
from brickie.kinds.seed import Seed
|
|
8
|
+
from brickie.kinds.source import Source
|
|
9
|
+
from brickie.load.project import Project
|
|
10
|
+
from brickie.names import LogicalName
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def check(project: Project) -> list[Diagnostic]:
|
|
14
|
+
known = _tables(project)
|
|
15
|
+
views = _materialized_views(project)
|
|
16
|
+
diagnostics = []
|
|
17
|
+
reads: dict[LogicalName, set[LogicalName]] = {}
|
|
18
|
+
for model in project.models:
|
|
19
|
+
file = model.sql_file.relative_to(project.root).as_posix()
|
|
20
|
+
try:
|
|
21
|
+
query = sql.parse(model.query)
|
|
22
|
+
except sql.Unparseable as failure:
|
|
23
|
+
diagnostics.append(_unparseable(file, failure))
|
|
24
|
+
continue
|
|
25
|
+
# A loop across Domains needs a read of another Domain's gold, which is its own failure.
|
|
26
|
+
reads[_gold(project, model)] = {
|
|
27
|
+
read for read in query.reads if read.domain == model.domain.lower()
|
|
28
|
+
}
|
|
29
|
+
diagnostics += _reads(project, known, file, query)
|
|
30
|
+
diagnostics += _build_order(project, known, model, file, query)
|
|
31
|
+
diagnostics += [
|
|
32
|
+
_stream_of_materialized_view(file, query.streams[read])
|
|
33
|
+
for read in sorted(query.streams.keys() & views, key=str)
|
|
34
|
+
]
|
|
35
|
+
diagnostics += [_unqualified(file, name) for name in query.unqualified]
|
|
36
|
+
diagnostics += [_computed(file, computed) for computed in query.computed]
|
|
37
|
+
diagnostics += [_file_read(file, found) for found in query.file_reads]
|
|
38
|
+
diagnostics += [_raw_placeholder(file, found) for found in query.raw_placeholders]
|
|
39
|
+
return (
|
|
40
|
+
diagnostics
|
|
41
|
+
+ _cycles(project, reads)
|
|
42
|
+
+ _seeds_named_like_sources(project)
|
|
43
|
+
+ _named_like_a_quarantine(project)
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _reads(project: Project, known: set[LogicalName], file: str, query: sql.Query):
|
|
48
|
+
"""Every read must be a table in the project. A real table name gets its Logical Name."""
|
|
49
|
+
diagnostics = []
|
|
50
|
+
for read, written in sorted(query.reads.items(), key=lambda item: str(item[0])):
|
|
51
|
+
if read in known:
|
|
52
|
+
continue
|
|
53
|
+
logical = names.logical_name_for(project, read)
|
|
54
|
+
if logical in known:
|
|
55
|
+
diagnostics.append(_real_name(file, written, logical))
|
|
56
|
+
elif logical:
|
|
57
|
+
diagnostics.append(_real_name_of_no_table(file, written, logical))
|
|
58
|
+
else:
|
|
59
|
+
diagnostics.append(_unresolved(file, written))
|
|
60
|
+
return diagnostics
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _build_order(
|
|
64
|
+
project: Project, known: set[LogicalName], model: Model, file: str, query: sql.Query
|
|
65
|
+
) -> list[Diagnostic]:
|
|
66
|
+
"""A Gold Model reads silver and gold, never bronze. Its own Domain's are all open to it, but
|
|
67
|
+
from other Domains it reads only Exported silver and the Shared Domain's gold.
|
|
68
|
+
|
|
69
|
+
So every read points to a step built earlier, and Domains can't read each other in a loop.
|
|
70
|
+
"""
|
|
71
|
+
bronze, silver = (layer.lower() for layer in project.platform.source_layers)
|
|
72
|
+
domain = model.domain.lower()
|
|
73
|
+
published = _published(project, known, domain)
|
|
74
|
+
upserts = {
|
|
75
|
+
LogicalName(source.domain, silver, source.name)
|
|
76
|
+
for source in project.sources
|
|
77
|
+
if source.silver.keys
|
|
78
|
+
}
|
|
79
|
+
diagnostics = []
|
|
80
|
+
for read, written in sorted(query.reads.items(), key=lambda item: str(item[0])):
|
|
81
|
+
if read not in known:
|
|
82
|
+
continue # Reported as a table that isn't in the project.
|
|
83
|
+
if read.layer == bronze:
|
|
84
|
+
diagnostics.append(_bronze_read(file, written, read, silver))
|
|
85
|
+
elif read.domain != domain and read not in published:
|
|
86
|
+
if read.layer == silver:
|
|
87
|
+
diagnostics.append(_private_silver(file, written, read))
|
|
88
|
+
else:
|
|
89
|
+
diagnostics.append(_other_domains_gold(file, written, read))
|
|
90
|
+
elif read in query.streams and read in upserts:
|
|
91
|
+
diagnostics.append(_stream_of_upsert(file, query.streams[read]))
|
|
92
|
+
return diagnostics
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _published(project: Project, known: set[LogicalName], domain: str) -> set[LogicalName]:
|
|
96
|
+
"""What a Gold Model in `domain` may read from other Domains."""
|
|
97
|
+
silver = project.platform.source_layers[1]
|
|
98
|
+
tables = (*project.sources, *project.seeds)
|
|
99
|
+
published = {
|
|
100
|
+
LogicalName(table.domain, silver, table.name) for table in tables if _is_exported(table)
|
|
101
|
+
}
|
|
102
|
+
if domain != SHARED_DOMAIN:
|
|
103
|
+
model_layers = {layer.lower() for layer in project.platform.model_layers}
|
|
104
|
+
published |= {
|
|
105
|
+
name for name in known if name.domain == SHARED_DOMAIN and name.layer in model_layers
|
|
106
|
+
}
|
|
107
|
+
return published
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _is_exported(table: Source | Seed) -> bool:
|
|
111
|
+
return table.silver.export if isinstance(table, Source) else table.export
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _materialized_views(project: Project) -> set[LogicalName]:
|
|
115
|
+
"""Every table no stream may read: each Seed, and each Gold Model that reads no stream.
|
|
116
|
+
|
|
117
|
+
A Gold Model whose SQL can't be read is left out, because it's reported on its own.
|
|
118
|
+
"""
|
|
119
|
+
silver = project.platform.source_layers[1]
|
|
120
|
+
views = {LogicalName(seed.domain, silver, seed.name) for seed in project.seeds}
|
|
121
|
+
for model in project.models:
|
|
122
|
+
try:
|
|
123
|
+
streaming = sql.streams(model.query)
|
|
124
|
+
except sql.Unparseable:
|
|
125
|
+
continue
|
|
126
|
+
if not streaming:
|
|
127
|
+
views.add(_gold(project, model))
|
|
128
|
+
return views
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _stream_of_materialized_view(file: str, written: sql.Found) -> Diagnostic:
|
|
132
|
+
return error(
|
|
133
|
+
"BRK212",
|
|
134
|
+
file,
|
|
135
|
+
f"{written.text} is a materialized view, which Databricks can recompute in full, and"
|
|
136
|
+
" `STREAM(...)` reads only new rows. The pipeline would fail when it starts.",
|
|
137
|
+
f"Read it without `STREAM(...)`, as `FROM {written.text}`, so this Gold Model refreshes"
|
|
138
|
+
" from the whole table.",
|
|
139
|
+
written.line,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _stream_of_upsert(file: str, written: sql.Found) -> Diagnostic:
|
|
144
|
+
return error(
|
|
145
|
+
"BRK211",
|
|
146
|
+
file,
|
|
147
|
+
f"{written.text} is upsert silver, which changes rows in place, and `STREAM(...)` reads"
|
|
148
|
+
" only new rows. The pipeline would fail at runtime, after its first update.",
|
|
149
|
+
f"Read it without `STREAM(...)`, as `FROM {written.text}`, so this Gold Model refreshes"
|
|
150
|
+
" from the whole table.",
|
|
151
|
+
written.line,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _bronze_read(file: str, written: sql.Found, read: LogicalName, silver: str) -> Diagnostic:
|
|
156
|
+
return error(
|
|
157
|
+
"BRK210",
|
|
158
|
+
file,
|
|
159
|
+
f"{written.text} is bronze, which only its silver reads. Bronze keeps rows as they"
|
|
160
|
+
" landed, so a Gold Model reads silver, built after it.",
|
|
161
|
+
f"Read {read.domain}.{silver}.{read.entity} instead.",
|
|
162
|
+
written.line,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _private_silver(file: str, written: sql.Found, read: LogicalName) -> Diagnostic:
|
|
167
|
+
return error(
|
|
168
|
+
"BRK209",
|
|
169
|
+
file,
|
|
170
|
+
f"{written.text} is silver the {read.domain} Domain hasn't Exported,"
|
|
171
|
+
" so only that Domain reads it.",
|
|
172
|
+
f"Ask the {read.domain} Domain's owner to Export it, with `export: true` under a"
|
|
173
|
+
" Source's `silver:` or in a Seed, or read a table it has Exported.",
|
|
174
|
+
written.line,
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _other_domains_gold(file: str, written: sql.Found, read: LogicalName) -> Diagnostic:
|
|
179
|
+
return error(
|
|
180
|
+
"BRK209",
|
|
181
|
+
file,
|
|
182
|
+
f"{written.text} is the {read.domain} Domain's gold. Nobody reads another Domain's gold,"
|
|
183
|
+
" so each Domain can change its own.",
|
|
184
|
+
f"Build it from silver the {read.domain} Domain has Exported, or move what's shared"
|
|
185
|
+
" into the Shared Domain.",
|
|
186
|
+
written.line,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _gold(project: Project, model: Model) -> LogicalName:
|
|
191
|
+
return LogicalName(model.domain, model.layer, model.name)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _cycles(project: Project, reads: dict[LogicalName, set[LogicalName]]) -> list[Diagnostic]:
|
|
195
|
+
"""Each loop of Gold Models reading each other, found once by a depth-first walk."""
|
|
196
|
+
files = {
|
|
197
|
+
_gold(project, model): model.sql_file.relative_to(project.root).as_posix()
|
|
198
|
+
for model in project.models
|
|
199
|
+
}
|
|
200
|
+
finished: set[LogicalName] = set()
|
|
201
|
+
diagnostics = []
|
|
202
|
+
|
|
203
|
+
def walk(model: LogicalName, path: list[LogicalName]) -> None:
|
|
204
|
+
path.append(model)
|
|
205
|
+
for read in sorted(reads[model] & reads.keys(), key=_dotted):
|
|
206
|
+
if read in path:
|
|
207
|
+
cycle = path[path.index(read) :]
|
|
208
|
+
first = cycle.index(min(cycle, key=_dotted))
|
|
209
|
+
diagnostics.append(_cycle(files, cycle[first:] + cycle[:first]))
|
|
210
|
+
elif read not in finished:
|
|
211
|
+
walk(read, path)
|
|
212
|
+
path.pop()
|
|
213
|
+
finished.add(model)
|
|
214
|
+
|
|
215
|
+
for model in sorted(reads, key=_dotted):
|
|
216
|
+
if model not in finished:
|
|
217
|
+
walk(model, [])
|
|
218
|
+
return sorted(diagnostics, key=lambda diagnostic: diagnostic.file)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _cycle(files: dict[LogicalName, str], cycle: list[LogicalName]) -> Diagnostic:
|
|
222
|
+
loop = " → ".join(_dotted(model) for model in [*cycle, cycle[0]])
|
|
223
|
+
return error(
|
|
224
|
+
"BRK208",
|
|
225
|
+
files[cycle[0]],
|
|
226
|
+
"A Gold Model can't read itself, directly or through other Gold Models,"
|
|
227
|
+
f" because each must be built before it's read: {loop}.",
|
|
228
|
+
"Remove one of these reads, so each Gold Model reads only tables built before it.",
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _tables(project: Project) -> set[LogicalName]:
|
|
233
|
+
tables = set()
|
|
234
|
+
for source in project.sources:
|
|
235
|
+
for layer in project.platform.source_layers:
|
|
236
|
+
tables.add(LogicalName(source.domain, layer, source.name))
|
|
237
|
+
if source.silver.quarantining:
|
|
238
|
+
silver = project.platform.source_layers[1]
|
|
239
|
+
tables.add(LogicalName(source.domain, silver, source.quarantine))
|
|
240
|
+
for seed in project.seeds:
|
|
241
|
+
tables.add(LogicalName(seed.domain, project.platform.source_layers[1], seed.name))
|
|
242
|
+
for model in project.models:
|
|
243
|
+
tables.add(_gold(project, model))
|
|
244
|
+
return tables
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _seeds_named_like_sources(project: Project) -> list[Diagnostic]:
|
|
248
|
+
silver = project.platform.source_layers[1]
|
|
249
|
+
sources = {(source.domain, source.name) for source in project.sources}
|
|
250
|
+
return [
|
|
251
|
+
error(
|
|
252
|
+
"BRK202",
|
|
253
|
+
seed.config_file.relative_to(project.root).as_posix(),
|
|
254
|
+
f"{seed.domain}.{silver}.{seed.name} is both a Seed and a Source.",
|
|
255
|
+
"Rename the Seed or the Source, so each has its own table.",
|
|
256
|
+
)
|
|
257
|
+
for seed in project.seeds
|
|
258
|
+
if (seed.domain, seed.name) in sources
|
|
259
|
+
]
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _named_like_a_quarantine(project: Project) -> list[Diagnostic]:
|
|
263
|
+
"""A Source's Quarantine table is in silver's schema, so no other table may take its name."""
|
|
264
|
+
silver = project.platform.source_layers[1]
|
|
265
|
+
quarantines = {
|
|
266
|
+
(source.domain, source.quarantine): source.name
|
|
267
|
+
for source in project.sources
|
|
268
|
+
if source.silver.quarantining
|
|
269
|
+
}
|
|
270
|
+
return [
|
|
271
|
+
error(
|
|
272
|
+
"BRK202",
|
|
273
|
+
table.config_file.relative_to(project.root).as_posix(),
|
|
274
|
+
f"{table.domain}.{silver}.{table.name} is also the Quarantine table"
|
|
275
|
+
f" of the Source {quarantines[(table.domain, table.name)]}.",
|
|
276
|
+
"Rename this file, so each table has its own name.",
|
|
277
|
+
)
|
|
278
|
+
for table in (*project.sources, *project.seeds)
|
|
279
|
+
if (table.domain, table.name) in quarantines
|
|
280
|
+
]
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _unresolved(file: str, read: sql.Found) -> Diagnostic:
|
|
284
|
+
return error(
|
|
285
|
+
"BRK201",
|
|
286
|
+
file,
|
|
287
|
+
f"{read.text} isn't a table in this project.",
|
|
288
|
+
"Declare a Source, Seed or Gold Model with this name, or correct the Logical Name.",
|
|
289
|
+
read.line,
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _real_name(file: str, real: sql.Found, logical: LogicalName) -> Diagnostic:
|
|
294
|
+
return error(
|
|
295
|
+
"BRK205",
|
|
296
|
+
file,
|
|
297
|
+
f"{real.text} is a real table name, which names one Environment.",
|
|
298
|
+
f"Use its Logical Name, {_dotted(logical)}, so the same SQL works in every Environment.",
|
|
299
|
+
real.line,
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _real_name_of_no_table(file: str, real: sql.Found, logical: LogicalName) -> Diagnostic:
|
|
304
|
+
return error(
|
|
305
|
+
"BRK205",
|
|
306
|
+
file,
|
|
307
|
+
f"{real.text} is a real table name, and no table called {_dotted(logical)}"
|
|
308
|
+
" is in this project.",
|
|
309
|
+
f"Declare a Source, Seed or Gold Model called {_dotted(logical)}, or correct the name.",
|
|
310
|
+
real.line,
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _file_read(file: str, found: sql.Found) -> Diagnostic:
|
|
315
|
+
return error(
|
|
316
|
+
"BRK206",
|
|
317
|
+
file,
|
|
318
|
+
f"{found.text} reads data that isn't a table in this project,"
|
|
319
|
+
" so Brickie can't check it or point it at each Environment.",
|
|
320
|
+
"A Gold Model reads tables. Declare a Source for these files"
|
|
321
|
+
" and read its silver table by Logical Name.",
|
|
322
|
+
found.line,
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _raw_placeholder(file: str, found: sql.Found) -> Diagnostic:
|
|
327
|
+
return error(
|
|
328
|
+
"BRK207",
|
|
329
|
+
file,
|
|
330
|
+
f"{found.text} holds `${{`, so the pipeline could replace it with a setting's value,"
|
|
331
|
+
" and a raw string can't hold the escape that stops it.",
|
|
332
|
+
"Write it as an ordinary string, doubling each backslash, and Brickie escapes the `${`.",
|
|
333
|
+
found.line,
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _dotted(name: LogicalName) -> str:
|
|
338
|
+
return f"{name.domain}.{name.layer}.{name.entity}"
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _unqualified(file: str, name: sql.Found) -> Diagnostic:
|
|
342
|
+
return error(
|
|
343
|
+
"BRK201",
|
|
344
|
+
file,
|
|
345
|
+
f"`{name.text}` isn't a CTE or a Logical Name, so Brickie can't tell which table it reads.",
|
|
346
|
+
"Write the table as a three-part Logical Name, such as sales.silver.orders,"
|
|
347
|
+
" or define a CTE with this name.",
|
|
348
|
+
name.line,
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _unparseable(file: str, failure: sql.Unparseable) -> Diagnostic:
|
|
353
|
+
return error(
|
|
354
|
+
"BRK203",
|
|
355
|
+
file,
|
|
356
|
+
f"Brickie can't read this SQL, so it can't tell what it reads. {failure.reason}",
|
|
357
|
+
"Make the file one Databricks SQL query that runs on its own.",
|
|
358
|
+
failure.line,
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _computed(file: str, computed: sql.Found) -> Diagnostic:
|
|
363
|
+
return error(
|
|
364
|
+
"BRK204",
|
|
365
|
+
file,
|
|
366
|
+
f"{computed.text} builds a table name as the query runs,"
|
|
367
|
+
" so Brickie can't see what it reads.",
|
|
368
|
+
"Write the table as a Logical Name, such as sales.silver.orders,"
|
|
369
|
+
" so every read shows in review.",
|
|
370
|
+
computed.line,
|
|
371
|
+
)
|