brickie-core 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. brickie/__init__.py +0 -0
  2. brickie/build/__init__.py +0 -0
  3. brickie/build/bundle.py +126 -0
  4. brickie/build/compiler.py +57 -0
  5. brickie/check/__init__.py +0 -0
  6. brickie/check/platform_objects.py +72 -0
  7. brickie/check/rules.py +371 -0
  8. brickie/cli.py +586 -0
  9. brickie/conditions.py +41 -0
  10. brickie/deploy/__init__.py +0 -0
  11. brickie/deploy/databricks.py +146 -0
  12. brickie/deploy/databricks_cli_release.py +13 -0
  13. brickie/deploy/deploy.py +71 -0
  14. brickie/deploy/pinned_cli.py +185 -0
  15. brickie/diagnostics.py +75 -0
  16. brickie/examples/kettle/brickie.platform.yaml +21 -0
  17. brickie/examples/kettle/data/day-1/customers/customers-day-1.json +40 -0
  18. brickie/examples/kettle/data/day-1/orders/orders-day-1.json +150 -0
  19. brickie/examples/kettle/data/day-1/product_prices/product_prices-day-1.json +6 -0
  20. brickie/examples/kettle/data/day-2/customers/customers-day-2.json +2 -0
  21. brickie/examples/kettle/data/day-2/orders/orders-day-2.json +100 -0
  22. brickie/examples/kettle/data/day-2/product_prices/product_prices-day-2.json +1 -0
  23. brickie/examples/kettle/domains/finance/models/gold/fct_revenue.sql +10 -0
  24. brickie/examples/kettle/domains/finance/models/gold/fct_revenue.yaml +4 -0
  25. brickie/examples/kettle/domains/sales/models/gold/dim_product_prices.sql +10 -0
  26. brickie/examples/kettle/domains/sales/models/gold/dim_product_prices.yaml +4 -0
  27. brickie/examples/kettle/domains/sales/models/gold/fct_orders.sql +16 -0
  28. brickie/examples/kettle/domains/sales/models/gold/fct_orders.yaml +4 -0
  29. brickie/examples/kettle/domains/sales/seeds/regions.csv +5 -0
  30. brickie/examples/kettle/domains/sales/seeds/regions.yaml +7 -0
  31. brickie/examples/kettle/domains/sales/sources/customers.yaml +14 -0
  32. brickie/examples/kettle/domains/sales/sources/orders.yaml +10 -0
  33. brickie/examples/kettle/domains/sales/sources/product_prices.yaml +5 -0
  34. brickie/examples/kettle/domains/shared/models/gold/dim_customers.sql +9 -0
  35. brickie/examples/kettle/domains/shared/models/gold/dim_customers.yaml +4 -0
  36. brickie/examples/kettle/domains/shared/models/gold/dim_date.sql +9 -0
  37. brickie/examples/kettle/domains/shared/models/gold/dim_date.yaml +4 -0
  38. brickie/examples/kettle/gitignore +1 -0
  39. brickie/kinds/__init__.py +24 -0
  40. brickie/kinds/base.py +75 -0
  41. brickie/kinds/model.py +109 -0
  42. brickie/kinds/platform.py +443 -0
  43. brickie/kinds/seed.py +279 -0
  44. brickie/kinds/source.py +346 -0
  45. brickie/load/__init__.py +0 -0
  46. brickie/load/config.py +155 -0
  47. brickie/load/project.py +203 -0
  48. brickie/names.py +153 -0
  49. brickie/output.py +49 -0
  50. brickie/reference.py +351 -0
  51. brickie/registry.py +967 -0
  52. brickie/scaffold.py +116 -0
  53. brickie/skills.py +185 -0
  54. brickie/sql.py +261 -0
  55. brickie/statements.py +25 -0
  56. brickie/templates/ci/github/.github/workflows/brickie-merge.yaml +24 -0
  57. brickie/templates/ci/github/.github/workflows/brickie-pull-request.yaml +23 -0
  58. brickie/templates/project/brickie.platform.yaml +17 -0
  59. brickie/templates/project/domains/sales/models/gold/dim_date.sql +9 -0
  60. brickie/templates/project/domains/sales/models/gold/dim_date.yaml +4 -0
  61. brickie/templates/project/domains/sales/models/gold/fct_orders.sql +2 -0
  62. brickie/templates/project/domains/sales/models/gold/fct_orders.yaml +4 -0
  63. brickie/templates/project/domains/sales/sources/orders.yaml +5 -0
  64. brickie/templates/project/gitignore +1 -0
  65. brickie/templates/skills/brickie-add-model/SKILL.md +17 -0
  66. brickie/templates/skills/brickie-add-source/SKILL.md +16 -0
  67. brickie_core-0.1.0.dist-info/METADATA +90 -0
  68. brickie_core-0.1.0.dist-info/RECORD +71 -0
  69. brickie_core-0.1.0.dist-info/WHEEL +4 -0
  70. brickie_core-0.1.0.dist-info/entry_points.txt +2 -0
  71. brickie_core-0.1.0.dist-info/licenses/LICENSE +202 -0
brickie/__init__.py ADDED
File without changes
File without changes
@@ -0,0 +1,126 @@
1
+ """Writes `databricks.yaml`: the resources, plus the placeholder values for each target."""
2
+
3
+ import yaml
4
+
5
+ from brickie import names, sql
6
+ from brickie.load.project import Project
7
+
8
+ REFRESH_JOB = "refresh"
9
+
10
+ # One retry survives a one-off blip. An error a retry can't fix, such as a bad row in append-only
11
+ # bronze, then fails in about a minute instead of waiting through the default five retries.
12
+ UPDATE_RETRIES = {"pipelines.numUpdateRetryAttempts": "1"}
13
+
14
+
15
+ def render(project: Project) -> str:
16
+ platform = project.platform
17
+ variables = list(names.settings(project, next(iter(platform.hosts))))
18
+ pipelines = {}
19
+ for domain in platform.domains:
20
+ if any(item.domain == domain for item in (*project.sources, *project.seeds)):
21
+ pipelines[f"{domain}_ingest"] = _pipeline(project, domain, "ingest")
22
+ if any(model.domain == domain for model in project.models):
23
+ pipelines[f"{domain}_gold"] = _pipeline(project, domain, "gold")
24
+ upstream = _upstream(project, set(pipelines))
25
+ tasks = [_task(pipeline, upstream.get(pipeline, [])) for pipeline in pipelines]
26
+ return yaml.safe_dump(
27
+ {
28
+ "bundle": {"name": platform.name},
29
+ # Bundle sync skips gitignored files, and `brickie init` gitignores the package.
30
+ "sync": {"include": ["src/**"]},
31
+ "variables": {name: {"description": name} for name in variables},
32
+ "targets": {env: _target(project, env) for env in platform.hosts},
33
+ "resources": {
34
+ "pipelines": pipelines,
35
+ "jobs": {REFRESH_JOB: {"name": _name(project, "refresh"), "tasks": tasks}},
36
+ },
37
+ },
38
+ sort_keys=False,
39
+ )
40
+
41
+
42
+ def _name(project: Project, *parts: str) -> str:
43
+ """A pipeline or Job name, with the Environment in it, so Environments can share a workspace.
44
+
45
+ Databricks refuses a second pipeline with the same name.
46
+ """
47
+ return "_".join((project.platform.name, "${bundle.target}", *parts))
48
+
49
+
50
+ def _target(project: Project, env: str) -> dict:
51
+ """One Environment's workspace and placeholder values, and its Job's schedule if it has one."""
52
+ target: dict = {
53
+ "workspace": {"host": project.platform.hosts[env]},
54
+ "variables": names.settings(project, env),
55
+ }
56
+ schedule = project.platform.environments[env].schedule
57
+ if schedule:
58
+ paused = schedule.paused
59
+ if paused is None:
60
+ paused = env != project.platform.production_environment
61
+ target["resources"] = {
62
+ "jobs": {
63
+ REFRESH_JOB: {
64
+ "schedule": {
65
+ "quartz_cron_expression": schedule.cron,
66
+ "timezone_id": schedule.timezone,
67
+ "pause_status": "PAUSED" if paused else "UNPAUSED",
68
+ }
69
+ }
70
+ }
71
+ }
72
+ return target
73
+
74
+
75
+ def _pipeline(project: Project, domain: str, group: str) -> dict:
76
+ platform = project.platform
77
+ # Every table is written by its full name, so this is only the pipeline's default. Model
78
+ # Layers have no order, so the first by name keeps it the same however the list is ordered.
79
+ layer = platform.source_layers[0] if group == "ingest" else min(platform.model_layers)
80
+ keys = names.settings(project, next(iter(platform.hosts)))
81
+ return {
82
+ "name": _name(project, domain, group),
83
+ "catalog": f"${{var.{names.catalog_setting(domain, layer)}}}",
84
+ "schema": f"${{var.{names.schema_setting(domain, layer)}}}",
85
+ "serverless": True,
86
+ "channel": "CURRENT",
87
+ "libraries": [{"glob": {"include": f"src/{domain}/{group}/**"}}],
88
+ "configuration": {**{key: f"${{var.{key}}}" for key in keys}, **UPDATE_RETRIES},
89
+ }
90
+
91
+
92
+ def _upstream(project: Project, pipelines: set[str]) -> dict[str, list[str]]:
93
+ """The pipelines each gold pipeline reads from, so its task waits only for those.
94
+
95
+ Ingest pipelines read only files, so they wait for nothing. Silver comes from its Domain's
96
+ ingest pipeline, and gold of another Domain, which can only be the Shared Domain's, from its
97
+ gold pipeline.
98
+ """
99
+ source_layers = {layer.lower() for layer in project.platform.source_layers}
100
+ # Logical Names are lowercase, and pipelines are keyed by the Domain as the contract writes it.
101
+ domains = {domain.lower(): domain for domain in project.platform.domains}
102
+ upstream: dict[str, list[str]] = {}
103
+ for model in project.models:
104
+ try:
105
+ reads = sql.reads(model.query)
106
+ except sql.Unparseable:
107
+ continue # `brickie validate` reports it, and `brickie deploy` refuses it.
108
+ reader = f"{model.domain}_gold"
109
+ needed = upstream.setdefault(reader, [])
110
+ for read in sorted(reads, key=str):
111
+ group = "ingest" if read.layer in source_layers else "gold"
112
+ pipeline = f"{domains.get(read.domain, read.domain)}_{group}"
113
+ if pipeline in pipelines and pipeline != reader and pipeline not in needed:
114
+ needed.append(pipeline)
115
+ return upstream
116
+
117
+
118
+ def _task(pipeline: str, needs: list[str]) -> dict:
119
+ task: dict = {
120
+ "task_key": pipeline,
121
+ "max_retries": 0,
122
+ "pipeline_task": {"pipeline_id": f"${{resources.pipelines.{pipeline}.id}}"},
123
+ }
124
+ if needs:
125
+ task["depends_on"] = [{"task_key": needed} for needed in needs]
126
+ return task
@@ -0,0 +1,57 @@
1
+ """Turns a Project into generated code: one file per table group, same in every Environment."""
2
+
3
+ import shutil
4
+ from pathlib import Path
5
+
6
+ from brickie import names, sql
7
+ from brickie.build import bundle
8
+ from brickie.kinds import model, seed, source
9
+ from brickie.load.project import Project
10
+ from brickie.names import LogicalName
11
+
12
+ BUILD_FOLDER = "build"
13
+
14
+
15
+ def generated_files(project: Project) -> dict[Path, str]:
16
+ """Generated code by path relative to the package, using placeholders."""
17
+ files: dict[Path, str] = {}
18
+ bronze_layer, silver_layer = project.platform.source_layers
19
+ for item in project.sources:
20
+ bronze = names.placeholders(LogicalName(item.domain, bronze_layer, item.name))
21
+ silver = names.placeholders(LogicalName(item.domain, silver_layer, item.name))
22
+ quarantine = names.placeholders(LogicalName(item.domain, silver_layer, item.quarantine))
23
+ path = Path("src") / item.domain / "ingest" / f"{item.name}.sql"
24
+ files[path] = source.generate(
25
+ item, names.path_placeholder(item), bronze, silver, quarantine
26
+ )
27
+ for item in project.seeds:
28
+ target = names.placeholders(LogicalName(item.domain, silver_layer, item.name))
29
+ files[Path("src") / item.domain / "ingest" / f"{item.name}.sql"] = seed.generate(
30
+ item, target
31
+ )
32
+ for item in project.models:
33
+ layer = item.layer
34
+ target = names.placeholders(LogicalName(item.domain, layer, item.name))
35
+ path = Path("src") / item.domain / "gold" / f"{item.name}.sql"
36
+ query = sql.escape_placeholders(item.query)
37
+ rewritten = sql.rewrite(query, names.placeholders)
38
+ files[path] = model.generate(item, target, rewritten, sql.streams(item.query))
39
+ return files
40
+
41
+
42
+ def write_package(project: Project) -> Path:
43
+ package = project.root / BUILD_FOLDER
44
+ shutil.rmtree(package, ignore_errors=True)
45
+ package.mkdir()
46
+ for relative, content in generated_files(project).items():
47
+ target = package / relative
48
+ target.parent.mkdir(parents=True, exist_ok=True)
49
+ target.write_text(content)
50
+ (package / "databricks.yaml").write_text(bundle.render(project))
51
+ return package
52
+
53
+
54
+ def print_for(project: Project, env: str) -> str:
55
+ """Every generated file as runnable SQL for one Environment."""
56
+ code = "\n".join(generated_files(project).values())
57
+ return names.resolve(code, project, env)
File without changes
@@ -0,0 +1,72 @@
1
+ """Platform rules. `brickie validate --platform` runs these against one Environment's workspace.
2
+
3
+ Brickie never creates catalogs or schemas. Whoever owns the platform does, so a missing one fails
4
+ here, before a deploy, rather than halfway through one.
5
+ """
6
+
7
+ from typing import Protocol
8
+
9
+ from brickie import names
10
+ from brickie.diagnostics import Diagnostic, error
11
+ from brickie.kinds import platform
12
+ from brickie.load.project import Project
13
+
14
+
15
+ class UnityCatalog(Protocol):
16
+ """What the Databricks boundary reports about a workspace's catalogs and schemas."""
17
+
18
+ def catalogs(self, host: str) -> set[str]:
19
+ """The catalogs the signed-in identity can see."""
20
+ ...
21
+
22
+ def schemas(self, host: str, catalog: str) -> set[str]:
23
+ """The schemas in one catalog the signed-in identity can see."""
24
+ ...
25
+
26
+
27
+ def check(project: Project, env: str, workspace: UnityCatalog) -> list[Diagnostic]:
28
+ host = project.platform.hosts[env]
29
+ catalogs, schemas = names.platform_objects(project.platform, [env])
30
+ present = _lowercase(workspace.catalogs(host))
31
+ diagnostics = []
32
+ for catalog in catalogs:
33
+ needed = [schema for schema in schemas if schema.split(".")[0] == catalog]
34
+ if catalog.lower() not in present:
35
+ diagnostics.append(_missing_catalog(env, host, catalog, needed))
36
+ continue
37
+ found = _lowercase(workspace.schemas(host, catalog))
38
+ diagnostics += [
39
+ _missing_schema(env, host, schema)
40
+ for schema in needed
41
+ if schema.split(".")[1].lower() not in found
42
+ ]
43
+ return diagnostics
44
+
45
+
46
+ def _lowercase(found: set[str]) -> set[str]:
47
+ """Unity Catalog matches names whatever their case."""
48
+ return {name.lower() for name in found}
49
+
50
+
51
+ def _missing_catalog(env: str, host: str, catalog: str, schemas: list[str]) -> Diagnostic:
52
+ listed = ", ".join(f"`{schema}`" for schema in schemas)
53
+ return error(
54
+ "BRK301",
55
+ platform.FILE_NAME,
56
+ f"The catalog `{catalog}` isn't in {env}'s workspace, {host}, or the identity Brickie"
57
+ f" signed in as can't see it. The project's tables in {env} need it.",
58
+ f"Create the catalog `{catalog}` with the schemas {listed}, or grant the identity"
59
+ " `USE CATALOG` on it. Brickie never creates catalogs.",
60
+ )
61
+
62
+
63
+ def _missing_schema(env: str, host: str, schema: str) -> Diagnostic:
64
+ catalog, name = schema.split(".")
65
+ return error(
66
+ "BRK302",
67
+ platform.FILE_NAME,
68
+ f"The schema `{schema}` isn't in {env}'s workspace, {host}, or the identity Brickie"
69
+ f" signed in as can't see it. The project's tables in {env} need it.",
70
+ f"Create the schema `{name}` in the catalog `{catalog}`, or grant the identity"
71
+ " `USE SCHEMA` on it. Brickie never creates schemas.",
72
+ )
brickie/check/rules.py ADDED
@@ -0,0 +1,371 @@
1
+ """Project rules. `brickie validate` runs these with no workspace."""
2
+
3
+ from brickie import names, sql
4
+ from brickie.diagnostics import Diagnostic, error
5
+ from brickie.kinds.model import Model
6
+ from brickie.kinds.platform import SHARED_DOMAIN
7
+ from brickie.kinds.seed import Seed
8
+ from brickie.kinds.source import Source
9
+ from brickie.load.project import Project
10
+ from brickie.names import LogicalName
11
+
12
+
13
+ def check(project: Project) -> list[Diagnostic]:
14
+ known = _tables(project)
15
+ views = _materialized_views(project)
16
+ diagnostics = []
17
+ reads: dict[LogicalName, set[LogicalName]] = {}
18
+ for model in project.models:
19
+ file = model.sql_file.relative_to(project.root).as_posix()
20
+ try:
21
+ query = sql.parse(model.query)
22
+ except sql.Unparseable as failure:
23
+ diagnostics.append(_unparseable(file, failure))
24
+ continue
25
+ # A loop across Domains needs a read of another Domain's gold, which is its own failure.
26
+ reads[_gold(project, model)] = {
27
+ read for read in query.reads if read.domain == model.domain.lower()
28
+ }
29
+ diagnostics += _reads(project, known, file, query)
30
+ diagnostics += _build_order(project, known, model, file, query)
31
+ diagnostics += [
32
+ _stream_of_materialized_view(file, query.streams[read])
33
+ for read in sorted(query.streams.keys() & views, key=str)
34
+ ]
35
+ diagnostics += [_unqualified(file, name) for name in query.unqualified]
36
+ diagnostics += [_computed(file, computed) for computed in query.computed]
37
+ diagnostics += [_file_read(file, found) for found in query.file_reads]
38
+ diagnostics += [_raw_placeholder(file, found) for found in query.raw_placeholders]
39
+ return (
40
+ diagnostics
41
+ + _cycles(project, reads)
42
+ + _seeds_named_like_sources(project)
43
+ + _named_like_a_quarantine(project)
44
+ )
45
+
46
+
47
+ def _reads(project: Project, known: set[LogicalName], file: str, query: sql.Query):
48
+ """Every read must be a table in the project. A real table name gets its Logical Name."""
49
+ diagnostics = []
50
+ for read, written in sorted(query.reads.items(), key=lambda item: str(item[0])):
51
+ if read in known:
52
+ continue
53
+ logical = names.logical_name_for(project, read)
54
+ if logical in known:
55
+ diagnostics.append(_real_name(file, written, logical))
56
+ elif logical:
57
+ diagnostics.append(_real_name_of_no_table(file, written, logical))
58
+ else:
59
+ diagnostics.append(_unresolved(file, written))
60
+ return diagnostics
61
+
62
+
63
+ def _build_order(
64
+ project: Project, known: set[LogicalName], model: Model, file: str, query: sql.Query
65
+ ) -> list[Diagnostic]:
66
+ """A Gold Model reads silver and gold, never bronze. Its own Domain's are all open to it, but
67
+ from other Domains it reads only Exported silver and the Shared Domain's gold.
68
+
69
+ So every read points to a step built earlier, and Domains can't read each other in a loop.
70
+ """
71
+ bronze, silver = (layer.lower() for layer in project.platform.source_layers)
72
+ domain = model.domain.lower()
73
+ published = _published(project, known, domain)
74
+ upserts = {
75
+ LogicalName(source.domain, silver, source.name)
76
+ for source in project.sources
77
+ if source.silver.keys
78
+ }
79
+ diagnostics = []
80
+ for read, written in sorted(query.reads.items(), key=lambda item: str(item[0])):
81
+ if read not in known:
82
+ continue # Reported as a table that isn't in the project.
83
+ if read.layer == bronze:
84
+ diagnostics.append(_bronze_read(file, written, read, silver))
85
+ elif read.domain != domain and read not in published:
86
+ if read.layer == silver:
87
+ diagnostics.append(_private_silver(file, written, read))
88
+ else:
89
+ diagnostics.append(_other_domains_gold(file, written, read))
90
+ elif read in query.streams and read in upserts:
91
+ diagnostics.append(_stream_of_upsert(file, query.streams[read]))
92
+ return diagnostics
93
+
94
+
95
+ def _published(project: Project, known: set[LogicalName], domain: str) -> set[LogicalName]:
96
+ """What a Gold Model in `domain` may read from other Domains."""
97
+ silver = project.platform.source_layers[1]
98
+ tables = (*project.sources, *project.seeds)
99
+ published = {
100
+ LogicalName(table.domain, silver, table.name) for table in tables if _is_exported(table)
101
+ }
102
+ if domain != SHARED_DOMAIN:
103
+ model_layers = {layer.lower() for layer in project.platform.model_layers}
104
+ published |= {
105
+ name for name in known if name.domain == SHARED_DOMAIN and name.layer in model_layers
106
+ }
107
+ return published
108
+
109
+
110
+ def _is_exported(table: Source | Seed) -> bool:
111
+ return table.silver.export if isinstance(table, Source) else table.export
112
+
113
+
114
+ def _materialized_views(project: Project) -> set[LogicalName]:
115
+ """Every table no stream may read: each Seed, and each Gold Model that reads no stream.
116
+
117
+ A Gold Model whose SQL can't be read is left out, because it's reported on its own.
118
+ """
119
+ silver = project.platform.source_layers[1]
120
+ views = {LogicalName(seed.domain, silver, seed.name) for seed in project.seeds}
121
+ for model in project.models:
122
+ try:
123
+ streaming = sql.streams(model.query)
124
+ except sql.Unparseable:
125
+ continue
126
+ if not streaming:
127
+ views.add(_gold(project, model))
128
+ return views
129
+
130
+
131
+ def _stream_of_materialized_view(file: str, written: sql.Found) -> Diagnostic:
132
+ return error(
133
+ "BRK212",
134
+ file,
135
+ f"{written.text} is a materialized view, which Databricks can recompute in full, and"
136
+ " `STREAM(...)` reads only new rows. The pipeline would fail when it starts.",
137
+ f"Read it without `STREAM(...)`, as `FROM {written.text}`, so this Gold Model refreshes"
138
+ " from the whole table.",
139
+ written.line,
140
+ )
141
+
142
+
143
+ def _stream_of_upsert(file: str, written: sql.Found) -> Diagnostic:
144
+ return error(
145
+ "BRK211",
146
+ file,
147
+ f"{written.text} is upsert silver, which changes rows in place, and `STREAM(...)` reads"
148
+ " only new rows. The pipeline would fail at runtime, after its first update.",
149
+ f"Read it without `STREAM(...)`, as `FROM {written.text}`, so this Gold Model refreshes"
150
+ " from the whole table.",
151
+ written.line,
152
+ )
153
+
154
+
155
+ def _bronze_read(file: str, written: sql.Found, read: LogicalName, silver: str) -> Diagnostic:
156
+ return error(
157
+ "BRK210",
158
+ file,
159
+ f"{written.text} is bronze, which only its silver reads. Bronze keeps rows as they"
160
+ " landed, so a Gold Model reads silver, built after it.",
161
+ f"Read {read.domain}.{silver}.{read.entity} instead.",
162
+ written.line,
163
+ )
164
+
165
+
166
+ def _private_silver(file: str, written: sql.Found, read: LogicalName) -> Diagnostic:
167
+ return error(
168
+ "BRK209",
169
+ file,
170
+ f"{written.text} is silver the {read.domain} Domain hasn't Exported,"
171
+ " so only that Domain reads it.",
172
+ f"Ask the {read.domain} Domain's owner to Export it, with `export: true` under a"
173
+ " Source's `silver:` or in a Seed, or read a table it has Exported.",
174
+ written.line,
175
+ )
176
+
177
+
178
+ def _other_domains_gold(file: str, written: sql.Found, read: LogicalName) -> Diagnostic:
179
+ return error(
180
+ "BRK209",
181
+ file,
182
+ f"{written.text} is the {read.domain} Domain's gold. Nobody reads another Domain's gold,"
183
+ " so each Domain can change its own.",
184
+ f"Build it from silver the {read.domain} Domain has Exported, or move what's shared"
185
+ " into the Shared Domain.",
186
+ written.line,
187
+ )
188
+
189
+
190
+ def _gold(project: Project, model: Model) -> LogicalName:
191
+ return LogicalName(model.domain, model.layer, model.name)
192
+
193
+
194
+ def _cycles(project: Project, reads: dict[LogicalName, set[LogicalName]]) -> list[Diagnostic]:
195
+ """Each loop of Gold Models reading each other, found once by a depth-first walk."""
196
+ files = {
197
+ _gold(project, model): model.sql_file.relative_to(project.root).as_posix()
198
+ for model in project.models
199
+ }
200
+ finished: set[LogicalName] = set()
201
+ diagnostics = []
202
+
203
+ def walk(model: LogicalName, path: list[LogicalName]) -> None:
204
+ path.append(model)
205
+ for read in sorted(reads[model] & reads.keys(), key=_dotted):
206
+ if read in path:
207
+ cycle = path[path.index(read) :]
208
+ first = cycle.index(min(cycle, key=_dotted))
209
+ diagnostics.append(_cycle(files, cycle[first:] + cycle[:first]))
210
+ elif read not in finished:
211
+ walk(read, path)
212
+ path.pop()
213
+ finished.add(model)
214
+
215
+ for model in sorted(reads, key=_dotted):
216
+ if model not in finished:
217
+ walk(model, [])
218
+ return sorted(diagnostics, key=lambda diagnostic: diagnostic.file)
219
+
220
+
221
+ def _cycle(files: dict[LogicalName, str], cycle: list[LogicalName]) -> Diagnostic:
222
+ loop = " → ".join(_dotted(model) for model in [*cycle, cycle[0]])
223
+ return error(
224
+ "BRK208",
225
+ files[cycle[0]],
226
+ "A Gold Model can't read itself, directly or through other Gold Models,"
227
+ f" because each must be built before it's read: {loop}.",
228
+ "Remove one of these reads, so each Gold Model reads only tables built before it.",
229
+ )
230
+
231
+
232
+ def _tables(project: Project) -> set[LogicalName]:
233
+ tables = set()
234
+ for source in project.sources:
235
+ for layer in project.platform.source_layers:
236
+ tables.add(LogicalName(source.domain, layer, source.name))
237
+ if source.silver.quarantining:
238
+ silver = project.platform.source_layers[1]
239
+ tables.add(LogicalName(source.domain, silver, source.quarantine))
240
+ for seed in project.seeds:
241
+ tables.add(LogicalName(seed.domain, project.platform.source_layers[1], seed.name))
242
+ for model in project.models:
243
+ tables.add(_gold(project, model))
244
+ return tables
245
+
246
+
247
+ def _seeds_named_like_sources(project: Project) -> list[Diagnostic]:
248
+ silver = project.platform.source_layers[1]
249
+ sources = {(source.domain, source.name) for source in project.sources}
250
+ return [
251
+ error(
252
+ "BRK202",
253
+ seed.config_file.relative_to(project.root).as_posix(),
254
+ f"{seed.domain}.{silver}.{seed.name} is both a Seed and a Source.",
255
+ "Rename the Seed or the Source, so each has its own table.",
256
+ )
257
+ for seed in project.seeds
258
+ if (seed.domain, seed.name) in sources
259
+ ]
260
+
261
+
262
+ def _named_like_a_quarantine(project: Project) -> list[Diagnostic]:
263
+ """A Source's Quarantine table is in silver's schema, so no other table may take its name."""
264
+ silver = project.platform.source_layers[1]
265
+ quarantines = {
266
+ (source.domain, source.quarantine): source.name
267
+ for source in project.sources
268
+ if source.silver.quarantining
269
+ }
270
+ return [
271
+ error(
272
+ "BRK202",
273
+ table.config_file.relative_to(project.root).as_posix(),
274
+ f"{table.domain}.{silver}.{table.name} is also the Quarantine table"
275
+ f" of the Source {quarantines[(table.domain, table.name)]}.",
276
+ "Rename this file, so each table has its own name.",
277
+ )
278
+ for table in (*project.sources, *project.seeds)
279
+ if (table.domain, table.name) in quarantines
280
+ ]
281
+
282
+
283
+ def _unresolved(file: str, read: sql.Found) -> Diagnostic:
284
+ return error(
285
+ "BRK201",
286
+ file,
287
+ f"{read.text} isn't a table in this project.",
288
+ "Declare a Source, Seed or Gold Model with this name, or correct the Logical Name.",
289
+ read.line,
290
+ )
291
+
292
+
293
+ def _real_name(file: str, real: sql.Found, logical: LogicalName) -> Diagnostic:
294
+ return error(
295
+ "BRK205",
296
+ file,
297
+ f"{real.text} is a real table name, which names one Environment.",
298
+ f"Use its Logical Name, {_dotted(logical)}, so the same SQL works in every Environment.",
299
+ real.line,
300
+ )
301
+
302
+
303
+ def _real_name_of_no_table(file: str, real: sql.Found, logical: LogicalName) -> Diagnostic:
304
+ return error(
305
+ "BRK205",
306
+ file,
307
+ f"{real.text} is a real table name, and no table called {_dotted(logical)}"
308
+ " is in this project.",
309
+ f"Declare a Source, Seed or Gold Model called {_dotted(logical)}, or correct the name.",
310
+ real.line,
311
+ )
312
+
313
+
314
+ def _file_read(file: str, found: sql.Found) -> Diagnostic:
315
+ return error(
316
+ "BRK206",
317
+ file,
318
+ f"{found.text} reads data that isn't a table in this project,"
319
+ " so Brickie can't check it or point it at each Environment.",
320
+ "A Gold Model reads tables. Declare a Source for these files"
321
+ " and read its silver table by Logical Name.",
322
+ found.line,
323
+ )
324
+
325
+
326
+ def _raw_placeholder(file: str, found: sql.Found) -> Diagnostic:
327
+ return error(
328
+ "BRK207",
329
+ file,
330
+ f"{found.text} holds `${{`, so the pipeline could replace it with a setting's value,"
331
+ " and a raw string can't hold the escape that stops it.",
332
+ "Write it as an ordinary string, doubling each backslash, and Brickie escapes the `${`.",
333
+ found.line,
334
+ )
335
+
336
+
337
+ def _dotted(name: LogicalName) -> str:
338
+ return f"{name.domain}.{name.layer}.{name.entity}"
339
+
340
+
341
+ def _unqualified(file: str, name: sql.Found) -> Diagnostic:
342
+ return error(
343
+ "BRK201",
344
+ file,
345
+ f"`{name.text}` isn't a CTE or a Logical Name, so Brickie can't tell which table it reads.",
346
+ "Write the table as a three-part Logical Name, such as sales.silver.orders,"
347
+ " or define a CTE with this name.",
348
+ name.line,
349
+ )
350
+
351
+
352
+ def _unparseable(file: str, failure: sql.Unparseable) -> Diagnostic:
353
+ return error(
354
+ "BRK203",
355
+ file,
356
+ f"Brickie can't read this SQL, so it can't tell what it reads. {failure.reason}",
357
+ "Make the file one Databricks SQL query that runs on its own.",
358
+ failure.line,
359
+ )
360
+
361
+
362
+ def _computed(file: str, computed: sql.Found) -> Diagnostic:
363
+ return error(
364
+ "BRK204",
365
+ file,
366
+ f"{computed.text} builds a table name as the query runs,"
367
+ " so Brickie can't see what it reads.",
368
+ "Write the table as a Logical Name, such as sales.silver.orders,"
369
+ " so every read shows in review.",
370
+ computed.line,
371
+ )