tabledoc 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tabledoc/__init__.py +8 -0
- tabledoc/__main__.py +5 -0
- tabledoc/cli.py +78 -0
- tabledoc/dbt.py +182 -0
- tabledoc/description.py +207 -0
- tabledoc/files.py +128 -0
- tabledoc/git.py +43 -0
- tabledoc/legacy.py +87 -0
- tabledoc/lineage.py +150 -0
- tabledoc/load.py +186 -0
- tabledoc/meta.py +441 -0
- tabledoc/report.py +63 -0
- tabledoc/schemas/dataset/v1.json +114 -0
- tabledoc/schemas/legacy/dataset/v1.json +117 -0
- tabledoc/schemas/legacy/table/v1.json +56 -0
- tabledoc/schemas/table/v1.json +84 -0
- tabledoc/schemas/tabledoc/v1.json +145 -0
- tabledoc/schemas.py +47 -0
- tabledoc/shape.py +117 -0
- tabledoc/source.py +188 -0
- tabledoc-0.2.0.dist-info/METADATA +87 -0
- tabledoc-0.2.0.dist-info/RECORD +25 -0
- tabledoc-0.2.0.dist-info/WHEEL +4 -0
- tabledoc-0.2.0.dist-info/entry_points.txt +2 -0
- tabledoc-0.2.0.dist-info/licenses/LICENSE +21 -0
tabledoc/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""tabledoc: describe the tables of a dataset in YAML and Markdown, check it, and write it into a DuckLake catalog."""
|
|
2
|
+
|
|
3
|
+
from .description import Description, Options, read
|
|
4
|
+
from .files import File
|
|
5
|
+
from .load import Target, load
|
|
6
|
+
from .report import Finding, Level, Report, SpecError
|
|
7
|
+
|
|
8
|
+
__all__ = ["Description", "File", "Finding", "Level", "Options", "Report", "SpecError", "Target", "load", "read"]
|
tabledoc/__main__.py
ADDED
tabledoc/cli.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""tabledoc validate and tabledoc load."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from .description import Options, read
|
|
12
|
+
from .load import Target, load
|
|
13
|
+
from .report import Level, Report, SpecError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _version() -> str:
|
|
17
|
+
try:
|
|
18
|
+
return f"tabledoc {version('tabledoc')}"
|
|
19
|
+
except PackageNotFoundError:
|
|
20
|
+
return "tabledoc"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def parser() -> argparse.ArgumentParser:
|
|
24
|
+
p = argparse.ArgumentParser(prog="tabledoc", description="Check a dataset's description.")
|
|
25
|
+
p.add_argument("--version", action="version", version=_version())
|
|
26
|
+
commands = p.add_subparsers(dest="command", required=True)
|
|
27
|
+
v = commands.add_parser("validate", help="check the description")
|
|
28
|
+
_common(v)
|
|
29
|
+
w = commands.add_parser("load", help="check the description and write it into a DuckLake catalog's _meta schema")
|
|
30
|
+
_common(w)
|
|
31
|
+
w.add_argument("--ducklake", default=None, help="the DuckLake catalog (default: FDL_WORKING)")
|
|
32
|
+
w.add_argument(
|
|
33
|
+
"--data-path", default=None, help="where new data files go (default: FDL_DATA_PATH, else fdl clone's note)"
|
|
34
|
+
)
|
|
35
|
+
w.add_argument("--out", type=Path, default=None, help="also write dataset.json and assets/ for fdl commit here")
|
|
36
|
+
w.add_argument("--assets", type=Path, default=None, help="files of the asset columns, linked into --out's assets/")
|
|
37
|
+
w.add_argument("--unlisted", action="store_true", help="write the tables with no description as not published")
|
|
38
|
+
return p
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _common(p: argparse.ArgumentParser) -> None:
|
|
42
|
+
p.add_argument("--root", type=Path, default=Path.cwd(), help="the folder of tabledoc.yml (default: here)")
|
|
43
|
+
p.add_argument("--manifest", type=Path, default=None, help="dbt's manifest.json (default: target/manifest.json)")
|
|
44
|
+
p.add_argument("--profiles", type=Path, default=None, help="dbt's profiles.yml (default: DBT_PROFILES_DIR, else the root)")
|
|
45
|
+
p.add_argument("--quiet", action="store_true", help="hide info findings")
|
|
46
|
+
p.add_argument("--strict", action="store_true", help="fail on warnings too")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def main(argv: list[str] | None = None) -> int:
|
|
50
|
+
args = parser().parse_args(argv)
|
|
51
|
+
report = Report()
|
|
52
|
+
options = Options(args.root, args.manifest, args.profiles, with_source=False)
|
|
53
|
+
try:
|
|
54
|
+
if args.command == "validate":
|
|
55
|
+
read(options, report)
|
|
56
|
+
else:
|
|
57
|
+
catalog = args.ducklake or os.environ.get("FDL_WORKING")
|
|
58
|
+
if not catalog:
|
|
59
|
+
raise SpecError("give the catalog: --ducklake or FDL_WORKING")
|
|
60
|
+
data = args.data_path or os.environ.get("FDL_DATA_PATH")
|
|
61
|
+
if args.assets and not args.out:
|
|
62
|
+
raise SpecError("--assets links its files into --out's assets/; give --out too")
|
|
63
|
+
target = Target(catalog, data, args.out, args.assets, args.unlisted)
|
|
64
|
+
if load(options, target, report, strict=args.strict) is not None:
|
|
65
|
+
print(f"wrote {target.catalog}" + (f" and {target.out}" if target.out else ""), file=sys.stderr)
|
|
66
|
+
except SpecError as exc:
|
|
67
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
68
|
+
return 2
|
|
69
|
+
rendered = report.render(show_info=not args.quiet)
|
|
70
|
+
if rendered:
|
|
71
|
+
print(rendered, file=sys.stderr)
|
|
72
|
+
warnings = report.of(Level.WARNING)
|
|
73
|
+
print(f"error {len(report.of(Level.ERROR))} / warning {len(warnings)}", file=sys.stderr)
|
|
74
|
+
return 1 if report.failed or (args.strict and warnings) else 0
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == "__main__":
|
|
78
|
+
sys.exit(main())
|
tabledoc/dbt.py
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""Tables from a dbt project's target/manifest.json.
|
|
2
|
+
|
|
3
|
+
The tables are the enabled models materialized as table, view or incremental,
|
|
4
|
+
under their `schema` and `alias`. A model's description comes from its yml:
|
|
5
|
+
`description`, the columns' `description`, `entity`, `dimension` and
|
|
6
|
+
`granularity`, and the keys dbt does not have under `config.meta`. `metrics`
|
|
7
|
+
come from the semantic layer. The SQL is `compiled_code`.
|
|
8
|
+
|
|
9
|
+
`catalogs` maps the alias another dataset is attached under in the SQL to its
|
|
10
|
+
repository URL, read from the ATTACHes in profiles.yml, so `depends_on` can
|
|
11
|
+
name tables of other datasets.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
from collections.abc import Mapping
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from . import schemas
|
|
23
|
+
from .report import Report
|
|
24
|
+
from .source import Key, Table
|
|
25
|
+
|
|
26
|
+
MATERIALIZED = ("table", "view", "incremental")
|
|
27
|
+
#: The keys of a table file a model writes under config.meta.
|
|
28
|
+
TABLE_META = (
|
|
29
|
+
"title",
|
|
30
|
+
"keywords",
|
|
31
|
+
"primary_key",
|
|
32
|
+
"licenses",
|
|
33
|
+
"sources",
|
|
34
|
+
"temporal_coverage",
|
|
35
|
+
"spatial_coverage",
|
|
36
|
+
"queria",
|
|
37
|
+
"annotations",
|
|
38
|
+
)
|
|
39
|
+
COLUMN_META = ("title", "asset", "annotations")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _meta(node: dict[str, Any]) -> dict[str, Any]:
|
|
43
|
+
meta = (node.get("config") or {}).get("meta") or node.get("meta") or {}
|
|
44
|
+
return meta if isinstance(meta, dict) else {}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _typed(value: Any) -> Any:
|
|
48
|
+
"""dbt's entity or dimension as a table file writes it: its type, or {type, name}."""
|
|
49
|
+
if isinstance(value, dict):
|
|
50
|
+
return {k: value[k] for k in ("type", "name") if value.get(k)}
|
|
51
|
+
return value or None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _column(column: dict[str, Any]) -> dict[str, Any]:
|
|
55
|
+
meta = _meta(column)
|
|
56
|
+
semantic = {k: v for k in ("entity", "dimension") if (v := _typed(column.get(k))) is not None}
|
|
57
|
+
return {
|
|
58
|
+
"name": column.get("name"),
|
|
59
|
+
**({"description": column["description"]} if column.get("description") else {}),
|
|
60
|
+
**{k: meta[k] for k in COLUMN_META if k in meta},
|
|
61
|
+
**semantic,
|
|
62
|
+
**({"granularity": column["granularity"]} if column.get("granularity") else {}),
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _metrics(manifest: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
|
|
67
|
+
"""Each model's simple metrics, by the model's unique_id."""
|
|
68
|
+
models = {
|
|
69
|
+
sm.get("name"): (sm.get("depends_on") or {}).get("nodes") or []
|
|
70
|
+
for sm in (manifest.get("semantic_models") or {}).values()
|
|
71
|
+
}
|
|
72
|
+
found: dict[str, list[dict[str, Any]]] = {}
|
|
73
|
+
for metric in (manifest.get("metrics") or {}).values():
|
|
74
|
+
params = metric.get("type_params") or {}
|
|
75
|
+
aggregation = params.get("metric_aggregation_params") or {}
|
|
76
|
+
if metric.get("type") != "simple" or not aggregation.get("semantic_model"):
|
|
77
|
+
continue
|
|
78
|
+
entry = {
|
|
79
|
+
"name": metric.get("name"),
|
|
80
|
+
**({"agg": aggregation["agg"]} if aggregation.get("agg") else {}),
|
|
81
|
+
**({"expr": params["expr"]} if params.get("expr") else {}),
|
|
82
|
+
**({"description": metric["description"]} if metric.get("description") else {}),
|
|
83
|
+
}
|
|
84
|
+
for model in models.get(aggregation["semantic_model"], []):
|
|
85
|
+
found.setdefault(model, []).append(entry)
|
|
86
|
+
return found
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _yml_path(root: Path, node: dict[str, Any]) -> Path:
|
|
90
|
+
"""The yml that describes the model, else its SQL file."""
|
|
91
|
+
patch = node.get("patch_path")
|
|
92
|
+
written = patch.split("://", 1)[-1] if isinstance(patch, str) else node.get("original_file_path", "")
|
|
93
|
+
return root / written
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def tables(manifest: dict[str, Any], root: Path, report: Report) -> list[Table]:
|
|
97
|
+
"""The tables dbt builds, each with its description, SQL and the guide beside its SQL."""
|
|
98
|
+
metrics = _metrics(manifest)
|
|
99
|
+
found = []
|
|
100
|
+
for uid, node in sorted((manifest.get("nodes") or {}).items()):
|
|
101
|
+
config = node.get("config") or {}
|
|
102
|
+
if (
|
|
103
|
+
node.get("resource_type") != "model"
|
|
104
|
+
or config.get("enabled") is False
|
|
105
|
+
or config.get("materialized") not in MATERIALIZED
|
|
106
|
+
):
|
|
107
|
+
continue
|
|
108
|
+
meta = _meta(node)
|
|
109
|
+
columns = [_column(c) for c in (node.get("columns") or {}).values() if isinstance(c, dict)]
|
|
110
|
+
described = {
|
|
111
|
+
**({"description": node["description"]} if node.get("description") else {}),
|
|
112
|
+
**{k: meta[k] for k in TABLE_META if k in meta},
|
|
113
|
+
**({"columns": columns} if columns else {}),
|
|
114
|
+
**({"metrics": metrics[uid]} if uid in metrics else {}),
|
|
115
|
+
}
|
|
116
|
+
path = _yml_path(root, node)
|
|
117
|
+
schemas.check({"$schema": schemas.TABLE, **described}, schemas.TABLE, path, report)
|
|
118
|
+
code = node.get("compiled_code")
|
|
119
|
+
sql_file = root / node.get("original_file_path", "")
|
|
120
|
+
guide = sql_file.with_suffix(".md")
|
|
121
|
+
found.append(
|
|
122
|
+
Table(
|
|
123
|
+
str(node.get("schema", "")),
|
|
124
|
+
str(node.get("alias") or node.get("name", "")),
|
|
125
|
+
record={
|
|
126
|
+
**described,
|
|
127
|
+
"materialized": config["materialized"],
|
|
128
|
+
**({"sql": code.strip()} if isinstance(code, str) and code.strip() else {}),
|
|
129
|
+
**({"source_path": node["original_file_path"]} if node.get("original_file_path") else {}),
|
|
130
|
+
},
|
|
131
|
+
path=path,
|
|
132
|
+
guide=guide.relative_to(root).as_posix() if guide.is_file() else None,
|
|
133
|
+
)
|
|
134
|
+
)
|
|
135
|
+
return found
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
_ENV_VAR = re.compile(r"""\{\{\s*env_var\(\s*(['"])(.+?)\1\s*(?:,\s*(['"])(.*?)\3\s*)?\)\s*\}\}""")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _render(text: str, env: Mapping[str, str]) -> str | None:
|
|
142
|
+
"""``text`` with `{{ env_var('NAME', 'default') }}` replaced. None when a value is missing."""
|
|
143
|
+
if any(m.group(2) not in env and m.group(4) is None for m in _ENV_VAR.finditer(text)):
|
|
144
|
+
return None
|
|
145
|
+
rendered = _ENV_VAR.sub(lambda m: env.get(m.group(2), m.group(4) or ""), text)
|
|
146
|
+
return None if "{" in rendered else rendered
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
_CATALOG_URL = re.compile(r"(https?://.+?/)(?:versions/[^/]+/)?(?:metadata\.db|ducklake\.duckdb)")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _repository(path: str, env: Mapping[str, str]) -> str | None:
|
|
153
|
+
"""The repository URL a DuckLake ATTACH path reads."""
|
|
154
|
+
rendered = _render(path, env)
|
|
155
|
+
match = _CATALOG_URL.fullmatch(rendered.removeprefix("ducklake:")) if rendered else None
|
|
156
|
+
return match.group(1) if match else None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _target(profile: dict[str, Any], env: Mapping[str, str]) -> dict[str, Any] | None:
|
|
160
|
+
name = env.get("DBT_TARGET") or _render(str(profile.get("target") or "default"), env)
|
|
161
|
+
target = (profile.get("outputs") or {}).get(name)
|
|
162
|
+
return target if isinstance(target, dict) else None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def catalogs(profiles: dict[str, Any], env: Mapping[str, str] | None = None) -> dict[str, str]:
|
|
166
|
+
"""Alias -> repository URL of each other dataset the profiles attach."""
|
|
167
|
+
env = os.environ if env is None else env
|
|
168
|
+
return {
|
|
169
|
+
str(attach["alias"]): repository
|
|
170
|
+
for name, profile in profiles.items()
|
|
171
|
+
if name != "config"
|
|
172
|
+
and isinstance(profile, dict)
|
|
173
|
+
and (target := _target(profile, env)) is not None
|
|
174
|
+
for attach in target.get("attach") or []
|
|
175
|
+
if isinstance(attach, dict)
|
|
176
|
+
and attach.get("alias")
|
|
177
|
+
and (repository := _repository(str(attach.get("path", "")), env)) is not None
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def keys(found: list[Table]) -> set[Key]:
|
|
182
|
+
return {t.key for t in found}
|
tabledoc/description.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""A dataset's description, read from what an author wrote under the root."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import yaml
|
|
11
|
+
|
|
12
|
+
from . import dbt, files, git, lineage, shape, source
|
|
13
|
+
from .report import Report
|
|
14
|
+
|
|
15
|
+
README = "README.md"
|
|
16
|
+
DEFAULT_MANIFEST = Path("target") / "manifest.json"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class Options:
|
|
21
|
+
root: Path
|
|
22
|
+
#: dbt's manifest.json; default target/manifest.json in a dbt project.
|
|
23
|
+
manifest: Path | None = None
|
|
24
|
+
#: dbt's profiles.yml, for the other datasets the SQL reads; default DBT_PROFILES_DIR or the root.
|
|
25
|
+
profiles: Path | None = None
|
|
26
|
+
#: Whether to say where it came from (`source`). Off for comparing outputs.
|
|
27
|
+
with_source: bool = True
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class Description:
|
|
32
|
+
"""What the author wrote, with the older formats read as today's.
|
|
33
|
+
|
|
34
|
+
Records are plain dicts and lists, keys in a fixed order. Guides and the
|
|
35
|
+
files they link to are named by path; ``files`` lists each with its sha256,
|
|
36
|
+
and ``content`` gives what to send for one.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
root: Path
|
|
40
|
+
dataset: dict[str, Any]
|
|
41
|
+
schemas: list[dict[str, Any]]
|
|
42
|
+
tables: list[dict[str, Any]]
|
|
43
|
+
files: list[files.File]
|
|
44
|
+
#: The Git working copy around the root; None outside Git or without ``with_source``.
|
|
45
|
+
source: dict[str, Any] | None = None
|
|
46
|
+
#: Alias -> repository URL of each other dataset the SQL reads (from dbt's profiles.yml).
|
|
47
|
+
catalogs: dict[str, str] | None = None
|
|
48
|
+
|
|
49
|
+
def content(self, file: files.File) -> bytes:
|
|
50
|
+
return files.content(self.root, file.path)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _profiles(options: Options) -> dict[str, Any]:
|
|
54
|
+
import os
|
|
55
|
+
|
|
56
|
+
folder = os.environ.get("DBT_PROFILES_DIR")
|
|
57
|
+
path = options.profiles or (Path(folder) if folder else options.root) / "profiles.yml"
|
|
58
|
+
if not path.is_file():
|
|
59
|
+
return {}
|
|
60
|
+
loaded = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
61
|
+
return loaded if isinstance(loaded, dict) else {}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _dbt_tables(options: Options, report: Report) -> tuple[list[source.Table], dict[str, str]]:
|
|
65
|
+
root = options.root
|
|
66
|
+
if not (root / "dbt_project.yml").is_file():
|
|
67
|
+
return [], {}
|
|
68
|
+
path = options.manifest or root / DEFAULT_MANIFEST
|
|
69
|
+
if not path.is_file():
|
|
70
|
+
report.warning(
|
|
71
|
+
"manifest-missing",
|
|
72
|
+
f"{path} is not here, so dbt's models are not read; run dbt compile first",
|
|
73
|
+
root / "dbt_project.yml",
|
|
74
|
+
)
|
|
75
|
+
return [], dbt.catalogs(_profiles(options))
|
|
76
|
+
manifest = json.loads(path.read_text(encoding="utf-8"))
|
|
77
|
+
return dbt.tables(manifest, root, report), dbt.catalogs(_profiles(options))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _merge(
|
|
81
|
+
built: list[source.Table], written: list[source.Table], report: Report
|
|
82
|
+
) -> list[source.Table]:
|
|
83
|
+
"""One table per (schema, name). A table is described in one place only."""
|
|
84
|
+
by_key: dict[source.Key, source.Table] = {}
|
|
85
|
+
for table in written:
|
|
86
|
+
if table.key in by_key:
|
|
87
|
+
report.error(
|
|
88
|
+
"described-twice",
|
|
89
|
+
f"{table.schema}.{table.name} is also described in {by_key[table.key].path}",
|
|
90
|
+
table.path,
|
|
91
|
+
)
|
|
92
|
+
continue
|
|
93
|
+
by_key[table.key] = table
|
|
94
|
+
merged = dict(by_key)
|
|
95
|
+
for model in built:
|
|
96
|
+
found = by_key.get(model.key)
|
|
97
|
+
if found is None:
|
|
98
|
+
merged[model.key] = model
|
|
99
|
+
continue
|
|
100
|
+
if shape.described(model.record):
|
|
101
|
+
report.error(
|
|
102
|
+
"described-twice",
|
|
103
|
+
f"{model.schema}.{model.name} is described in dbt's yml and in {found.path}; keep one",
|
|
104
|
+
model.path,
|
|
105
|
+
)
|
|
106
|
+
built = {k: model.record[k] for k in shape.BUILT_KEYS if k in model.record and k not in found.record}
|
|
107
|
+
merged[model.key] = source.Table(
|
|
108
|
+
found.schema,
|
|
109
|
+
found.name,
|
|
110
|
+
record={**found.record, **built},
|
|
111
|
+
path=found.path,
|
|
112
|
+
guide=found.guide or model.guide,
|
|
113
|
+
)
|
|
114
|
+
return [merged[k] for k in sorted(merged)]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _warn_metrics(tables: list[source.Table], report: Report) -> None:
|
|
118
|
+
for table in tables:
|
|
119
|
+
for metric in table.record.get("metrics") or []:
|
|
120
|
+
if isinstance(metric, dict) and not metric.get("agg"):
|
|
121
|
+
report.warning(
|
|
122
|
+
"metric-without-agg", f"{table.schema}.{table.name}: metric {metric.get('name')} has no agg", table.path
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _table(table: source.Table) -> dict[str, Any]:
|
|
127
|
+
shaped = shape.table(table.record)
|
|
128
|
+
head = {k: shaped[k] for k in shape.TABLE_KEYS if k in shaped}
|
|
129
|
+
tail = {k: v for k, v in shaped.items() if k not in shape.TABLE_KEYS}
|
|
130
|
+
return {
|
|
131
|
+
"schema": table.schema,
|
|
132
|
+
"name": table.name,
|
|
133
|
+
**head,
|
|
134
|
+
**({"guide": table.guide} if table.guide else {}),
|
|
135
|
+
**tail,
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _schemas(record: dict[str, Any]) -> list[dict[str, Any]]:
|
|
140
|
+
written = record.get("schemas")
|
|
141
|
+
return [
|
|
142
|
+
{**shape.pick(s, shape.SCHEMA_KEYS), **({"guide": files.normalized(s["guide"])} if s.get("guide") else {})}
|
|
143
|
+
for s in (written if isinstance(written, list) else [])
|
|
144
|
+
if isinstance(s, dict)
|
|
145
|
+
]
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _documents(
|
|
149
|
+
dataset: source.Dataset, schema_list: list[dict[str, Any]], tables: list[source.Table], root: Path, report: Report
|
|
150
|
+
) -> tuple[list[str], list[tuple[str, Path]]]:
|
|
151
|
+
"""The dataset's docs as normalized paths, and every document named, with where it is named."""
|
|
152
|
+
written = dataset.record.get("docs")
|
|
153
|
+
docs = []
|
|
154
|
+
for entry in written if isinstance(written, list) else []:
|
|
155
|
+
path = files.normalized(str(entry))
|
|
156
|
+
if path is None:
|
|
157
|
+
report.error("file-outside", f"docs: {entry} is outside {root}", dataset.path)
|
|
158
|
+
else:
|
|
159
|
+
docs.append(path)
|
|
160
|
+
named = [
|
|
161
|
+
*([(README, root / README)] if (root / README).is_file() else []),
|
|
162
|
+
*[(s["guide"], dataset.path) for s in schema_list if s.get("guide")],
|
|
163
|
+
*[(d, dataset.path) for d in docs],
|
|
164
|
+
*[(t.guide, t.path) for t in tables if t.guide],
|
|
165
|
+
]
|
|
166
|
+
for schema in schema_list:
|
|
167
|
+
if "guide" in schema and schema["guide"] is None:
|
|
168
|
+
report.error("file-outside", f"schemas: {schema['name']}'s guide is outside {root}", dataset.path)
|
|
169
|
+
return docs, [(p, w) for p, w in named if p]
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def read(options: Options, report: Report) -> Description:
|
|
173
|
+
"""Read and check the description under ``options.root``; findings go to ``report``."""
|
|
174
|
+
root = options.root.resolve()
|
|
175
|
+
options = Options(root, options.manifest, options.profiles, options.with_source)
|
|
176
|
+
dataset = source.read_dataset(root, report)
|
|
177
|
+
models, attached = _dbt_tables(options, report)
|
|
178
|
+
written = [*source.read_table_files(root, report), *source.read_legacy_tables(root, report)]
|
|
179
|
+
tables = _merge(models, written, report)
|
|
180
|
+
_warn_metrics(tables, report)
|
|
181
|
+
schema_list = _schemas(dataset.record)
|
|
182
|
+
docs, named = _documents(dataset, schema_list, tables, root, report)
|
|
183
|
+
sent = files.collect(root, named, report)
|
|
184
|
+
shaped = lineage.fill([_table(t) for t in tables], {**attached, **dataset.catalogs}, report)
|
|
185
|
+
record = dataset.record
|
|
186
|
+
return Description(
|
|
187
|
+
root=root,
|
|
188
|
+
dataset={
|
|
189
|
+
**shape.pick(record, shape.DATASET_KEYS),
|
|
190
|
+
**({"guide": README} if (root / README).is_file() else {}),
|
|
191
|
+
**({"docs": docs} if docs else {}),
|
|
192
|
+
**shape.annotations(record),
|
|
193
|
+
},
|
|
194
|
+
schemas=schema_list,
|
|
195
|
+
tables=[_ordered(t) for t in shaped],
|
|
196
|
+
files=sent,
|
|
197
|
+
source=git.source(root) if options.with_source else None,
|
|
198
|
+
catalogs={**attached, **dataset.catalogs} or None,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _ordered(table: dict[str, Any]) -> dict[str, Any]:
|
|
203
|
+
"""depends_on, filled in last, back in its place after sql."""
|
|
204
|
+
if "depends_on" not in table:
|
|
205
|
+
return table
|
|
206
|
+
order = ["schema", "name", *shape.TABLE_KEYS, "guide", "columns", "metrics", "annotations"]
|
|
207
|
+
return {k: table[k] for k in order if k in table}
|
tabledoc/files.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""The files whose content is sent: guides, README.md, docs, and what they link to.
|
|
2
|
+
|
|
3
|
+
Each is named by its path under the root, with its sha256. A
|
|
4
|
+
Markdown table file of the older format is sent without its front matter.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import posixpath
|
|
11
|
+
import re
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from .report import Report
|
|
17
|
+
from .source import legacy_front_matter
|
|
18
|
+
|
|
19
|
+
#: Relative links in Markdown: [text](target), , and src="target".
|
|
20
|
+
_LINK = re.compile(r"""(\]\(\s*<?)([^)\s>]+)|(\bsrc\s*=\s*["'])([^"']+)""")
|
|
21
|
+
|
|
22
|
+
#: Fixed, so the same file gets the same type on every machine.
|
|
23
|
+
MEDIA_TYPES = {
|
|
24
|
+
".md": "text/markdown",
|
|
25
|
+
".markdown": "text/markdown",
|
|
26
|
+
".txt": "text/plain",
|
|
27
|
+
".csv": "text/csv",
|
|
28
|
+
".tsv": "text/tab-separated-values",
|
|
29
|
+
".html": "text/html",
|
|
30
|
+
".htm": "text/html",
|
|
31
|
+
".json": "application/json",
|
|
32
|
+
".geojson": "application/geo+json",
|
|
33
|
+
".yml": "application/yaml",
|
|
34
|
+
".yaml": "application/yaml",
|
|
35
|
+
".xml": "application/xml",
|
|
36
|
+
".pdf": "application/pdf",
|
|
37
|
+
".zip": "application/zip",
|
|
38
|
+
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
39
|
+
".xls": "application/vnd.ms-excel",
|
|
40
|
+
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
41
|
+
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
42
|
+
".png": "image/png",
|
|
43
|
+
".jpg": "image/jpeg",
|
|
44
|
+
".jpeg": "image/jpeg",
|
|
45
|
+
".gif": "image/gif",
|
|
46
|
+
".svg": "image/svg+xml",
|
|
47
|
+
".webp": "image/webp",
|
|
48
|
+
".avif": "image/avif",
|
|
49
|
+
}
|
|
50
|
+
OCTET_STREAM = "application/octet-stream"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def media_type(path: str) -> str:
|
|
54
|
+
return MEDIA_TYPES.get(posixpath.splitext(path)[1].lower(), OCTET_STREAM)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def content(root: Path, path: str) -> bytes:
|
|
58
|
+
"""What is sent for ``path``: the file, or a Markdown table file's text below its front matter."""
|
|
59
|
+
file = root / path
|
|
60
|
+
if path.lower().endswith(".md"):
|
|
61
|
+
found = legacy_front_matter(file)
|
|
62
|
+
if found is not None:
|
|
63
|
+
return found[1].encode("utf-8")
|
|
64
|
+
return file.read_bytes()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _is_relative(target: str) -> bool:
|
|
68
|
+
return not (target.startswith(("#", "/")) or re.match(r"^[a-z][a-z0-9+.-]*:", target, re.IGNORECASE))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def links(text: str) -> list[str]:
|
|
72
|
+
"""The relative link targets in ``text``, without #fragment or ?query."""
|
|
73
|
+
found = (m.group(2) or m.group(4) for m in _LINK.finditer(text))
|
|
74
|
+
return [t for t in (re.split(r"[?#]", f)[0] for f in found if _is_relative(f)) if t]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def normalized(path: str) -> str | None:
|
|
78
|
+
"""``path`` normalized, or None when it leaves the root."""
|
|
79
|
+
clean = posixpath.normpath(path)
|
|
80
|
+
return None if clean == ".." or clean.startswith("../") or posixpath.isabs(clean) else clean
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True)
|
|
84
|
+
class File:
|
|
85
|
+
path: str
|
|
86
|
+
sha256: str
|
|
87
|
+
bytes: int
|
|
88
|
+
media_type: str
|
|
89
|
+
|
|
90
|
+
def entry(self) -> dict[str, Any]:
|
|
91
|
+
return {"path": self.path, "sha256": self.sha256, "bytes": self.bytes, "media_type": self.media_type}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def describe(root: Path, path: str) -> File:
|
|
95
|
+
body = content(root, path)
|
|
96
|
+
return File(path, hashlib.sha256(body).hexdigest(), len(body), media_type(path))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def collect(root: Path, documents: list[tuple[str, Path]], report: Report) -> list[File]:
|
|
100
|
+
"""``documents`` (path, the file that names it) and the files their Markdown links to.
|
|
101
|
+
|
|
102
|
+
A named document that is missing, or outside the root, is an error. A link
|
|
103
|
+
to a folder is left alone; a link to a missing file is a warning.
|
|
104
|
+
"""
|
|
105
|
+
seen: dict[str, File] = {}
|
|
106
|
+
pending = list(documents)
|
|
107
|
+
named = {path for path, _ in documents}
|
|
108
|
+
while pending:
|
|
109
|
+
path, where = pending.pop(0)
|
|
110
|
+
if path in seen:
|
|
111
|
+
continue
|
|
112
|
+
file = root / path
|
|
113
|
+
if not file.is_file() or not file.resolve().is_relative_to(root.resolve()):
|
|
114
|
+
if path in named:
|
|
115
|
+
report.error("file-missing", f"{path} is not a file under {root}", where)
|
|
116
|
+
elif not file.is_dir():
|
|
117
|
+
report.warning("link-broken", f"links to {path}, which is not here", where)
|
|
118
|
+
continue
|
|
119
|
+
seen[path] = describe(root, path)
|
|
120
|
+
if path.lower().endswith(".md"):
|
|
121
|
+
text = content(root, path).decode("utf-8", errors="replace")
|
|
122
|
+
for target in links(text):
|
|
123
|
+
linked = normalized(posixpath.join(posixpath.dirname(path), target))
|
|
124
|
+
if linked is None:
|
|
125
|
+
report.error("link-outside", f"links to {target}, outside {root}", file)
|
|
126
|
+
elif linked not in seen:
|
|
127
|
+
pending.append((linked, file))
|
|
128
|
+
return [seen[p] for p in sorted(seen)]
|
tabledoc/git.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Where a description came from: the Git working copy around the root."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import subprocess
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _git(root: Path, *args: str) -> str | None:
|
|
12
|
+
try:
|
|
13
|
+
done = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True, check=False)
|
|
14
|
+
except FileNotFoundError:
|
|
15
|
+
return None
|
|
16
|
+
return done.stdout.strip() if done.returncode == 0 else None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def https(url: str) -> str:
|
|
20
|
+
"""A remote URL as https, without credentials or `.git`: git@github.com:a/b.git -> https://github.com/a/b."""
|
|
21
|
+
scp = re.fullmatch(r"(?:[^@/]+@)?([^:/]+):(?!//)(.+)", url)
|
|
22
|
+
if scp:
|
|
23
|
+
url = f"https://{scp.group(1)}/{scp.group(2)}"
|
|
24
|
+
url = re.sub(r"^(?:ssh|git|http)://", "https://", url)
|
|
25
|
+
url = re.sub(r"^https://[^@/]+@", "https://", url)
|
|
26
|
+
return url.removesuffix("/").removesuffix(".git")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def source(root: Path) -> dict[str, Any] | None:
|
|
30
|
+
"""{repository, commit, path, dirty}, or None outside Git."""
|
|
31
|
+
top = _git(root, "rev-parse", "--show-toplevel")
|
|
32
|
+
commit = _git(root, "rev-parse", "HEAD")
|
|
33
|
+
if top is None or commit is None:
|
|
34
|
+
return None
|
|
35
|
+
remote = _git(root, "remote", "get-url", "origin")
|
|
36
|
+
status = _git(root, "status", "--porcelain")
|
|
37
|
+
path = root.resolve().relative_to(Path(top).resolve()).as_posix()
|
|
38
|
+
return {
|
|
39
|
+
**({"repository": https(remote)} if remote else {}),
|
|
40
|
+
"commit": commit,
|
|
41
|
+
"path": "" if path == "." else path,
|
|
42
|
+
"dirty": bool(status),
|
|
43
|
+
}
|