tabledoc 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tabledoc/__init__.py ADDED
@@ -0,0 +1,8 @@
1
+ """tabledoc: describe the tables of a dataset in YAML and Markdown, check it, and write it into a DuckLake catalog."""
2
+
3
+ from .description import Description, Options, read
4
+ from .files import File
5
+ from .load import Target, load
6
+ from .report import Finding, Level, Report, SpecError
7
+
8
+ __all__ = ["Description", "File", "Finding", "Level", "Options", "Report", "SpecError", "Target", "load", "read"]
tabledoc/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
tabledoc/cli.py ADDED
@@ -0,0 +1,78 @@
1
+ """tabledoc validate and tabledoc load."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import os
7
+ import sys
8
+ from importlib.metadata import PackageNotFoundError, version
9
+ from pathlib import Path
10
+
11
+ from .description import Options, read
12
+ from .load import Target, load
13
+ from .report import Level, Report, SpecError
14
+
15
+
16
+ def _version() -> str:
17
+ try:
18
+ return f"tabledoc {version('tabledoc')}"
19
+ except PackageNotFoundError:
20
+ return "tabledoc"
21
+
22
+
23
+ def parser() -> argparse.ArgumentParser:
24
+ p = argparse.ArgumentParser(prog="tabledoc", description="Check a dataset's description.")
25
+ p.add_argument("--version", action="version", version=_version())
26
+ commands = p.add_subparsers(dest="command", required=True)
27
+ v = commands.add_parser("validate", help="check the description")
28
+ _common(v)
29
+ w = commands.add_parser("load", help="check the description and write it into a DuckLake catalog's _meta schema")
30
+ _common(w)
31
+ w.add_argument("--ducklake", default=None, help="the DuckLake catalog (default: FDL_WORKING)")
32
+ w.add_argument(
33
+ "--data-path", default=None, help="where new data files go (default: FDL_DATA_PATH, else fdl clone's note)"
34
+ )
35
+ w.add_argument("--out", type=Path, default=None, help="also write dataset.json and assets/ for fdl commit here")
36
+ w.add_argument("--assets", type=Path, default=None, help="files of the asset columns, linked into --out's assets/")
37
+ w.add_argument("--unlisted", action="store_true", help="write the tables with no description as not published")
38
+ return p
39
+
40
+
41
+ def _common(p: argparse.ArgumentParser) -> None:
42
+ p.add_argument("--root", type=Path, default=Path.cwd(), help="the folder of tabledoc.yml (default: here)")
43
+ p.add_argument("--manifest", type=Path, default=None, help="dbt's manifest.json (default: target/manifest.json)")
44
+ p.add_argument("--profiles", type=Path, default=None, help="dbt's profiles.yml (default: DBT_PROFILES_DIR, else the root)")
45
+ p.add_argument("--quiet", action="store_true", help="hide info findings")
46
+ p.add_argument("--strict", action="store_true", help="fail on warnings too")
47
+
48
+
49
+ def main(argv: list[str] | None = None) -> int:
50
+ args = parser().parse_args(argv)
51
+ report = Report()
52
+ options = Options(args.root, args.manifest, args.profiles, with_source=False)
53
+ try:
54
+ if args.command == "validate":
55
+ read(options, report)
56
+ else:
57
+ catalog = args.ducklake or os.environ.get("FDL_WORKING")
58
+ if not catalog:
59
+ raise SpecError("give the catalog: --ducklake or FDL_WORKING")
60
+ data = args.data_path or os.environ.get("FDL_DATA_PATH")
61
+ if args.assets and not args.out:
62
+ raise SpecError("--assets links its files into --out's assets/; give --out too")
63
+ target = Target(catalog, data, args.out, args.assets, args.unlisted)
64
+ if load(options, target, report, strict=args.strict) is not None:
65
+ print(f"wrote {target.catalog}" + (f" and {target.out}" if target.out else ""), file=sys.stderr)
66
+ except SpecError as exc:
67
+ print(f"error: {exc}", file=sys.stderr)
68
+ return 2
69
+ rendered = report.render(show_info=not args.quiet)
70
+ if rendered:
71
+ print(rendered, file=sys.stderr)
72
+ warnings = report.of(Level.WARNING)
73
+ print(f"error {len(report.of(Level.ERROR))} / warning {len(warnings)}", file=sys.stderr)
74
+ return 1 if report.failed or (args.strict and warnings) else 0
75
+
76
+
77
+ if __name__ == "__main__":
78
+ sys.exit(main())
tabledoc/dbt.py ADDED
@@ -0,0 +1,182 @@
1
+ """Tables from a dbt project's target/manifest.json.
2
+
3
+ The tables are the enabled models materialized as table, view or incremental,
4
+ under their `schema` and `alias`. A model's description comes from its yml:
5
+ `description`, the columns' `description`, `entity`, `dimension` and
6
+ `granularity`, and the keys dbt does not have under `config.meta`. `metrics`
7
+ come from the semantic layer. The SQL is `compiled_code`.
8
+
9
+ `catalogs` maps the alias another dataset is attached under in the SQL to its
10
+ repository URL, read from the ATTACHes in profiles.yml, so `depends_on` can
11
+ name tables of other datasets.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import os
17
+ import re
18
+ from collections.abc import Mapping
19
+ from pathlib import Path
20
+ from typing import Any
21
+
22
+ from . import schemas
23
+ from .report import Report
24
+ from .source import Key, Table
25
+
26
+ MATERIALIZED = ("table", "view", "incremental")
27
+ #: The keys of a table file a model writes under config.meta.
28
+ TABLE_META = (
29
+ "title",
30
+ "keywords",
31
+ "primary_key",
32
+ "licenses",
33
+ "sources",
34
+ "temporal_coverage",
35
+ "spatial_coverage",
36
+ "queria",
37
+ "annotations",
38
+ )
39
+ COLUMN_META = ("title", "asset", "annotations")
40
+
41
+
42
+ def _meta(node: dict[str, Any]) -> dict[str, Any]:
43
+ meta = (node.get("config") or {}).get("meta") or node.get("meta") or {}
44
+ return meta if isinstance(meta, dict) else {}
45
+
46
+
47
+ def _typed(value: Any) -> Any:
48
+ """dbt's entity or dimension as a table file writes it: its type, or {type, name}."""
49
+ if isinstance(value, dict):
50
+ return {k: value[k] for k in ("type", "name") if value.get(k)}
51
+ return value or None
52
+
53
+
54
+ def _column(column: dict[str, Any]) -> dict[str, Any]:
55
+ meta = _meta(column)
56
+ semantic = {k: v for k in ("entity", "dimension") if (v := _typed(column.get(k))) is not None}
57
+ return {
58
+ "name": column.get("name"),
59
+ **({"description": column["description"]} if column.get("description") else {}),
60
+ **{k: meta[k] for k in COLUMN_META if k in meta},
61
+ **semantic,
62
+ **({"granularity": column["granularity"]} if column.get("granularity") else {}),
63
+ }
64
+
65
+
66
+ def _metrics(manifest: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
67
+ """Each model's simple metrics, by the model's unique_id."""
68
+ models = {
69
+ sm.get("name"): (sm.get("depends_on") or {}).get("nodes") or []
70
+ for sm in (manifest.get("semantic_models") or {}).values()
71
+ }
72
+ found: dict[str, list[dict[str, Any]]] = {}
73
+ for metric in (manifest.get("metrics") or {}).values():
74
+ params = metric.get("type_params") or {}
75
+ aggregation = params.get("metric_aggregation_params") or {}
76
+ if metric.get("type") != "simple" or not aggregation.get("semantic_model"):
77
+ continue
78
+ entry = {
79
+ "name": metric.get("name"),
80
+ **({"agg": aggregation["agg"]} if aggregation.get("agg") else {}),
81
+ **({"expr": params["expr"]} if params.get("expr") else {}),
82
+ **({"description": metric["description"]} if metric.get("description") else {}),
83
+ }
84
+ for model in models.get(aggregation["semantic_model"], []):
85
+ found.setdefault(model, []).append(entry)
86
+ return found
87
+
88
+
89
+ def _yml_path(root: Path, node: dict[str, Any]) -> Path:
90
+ """The yml that describes the model, else its SQL file."""
91
+ patch = node.get("patch_path")
92
+ written = patch.split("://", 1)[-1] if isinstance(patch, str) else node.get("original_file_path", "")
93
+ return root / written
94
+
95
+
96
+ def tables(manifest: dict[str, Any], root: Path, report: Report) -> list[Table]:
97
+ """The tables dbt builds, each with its description, SQL and the guide beside its SQL."""
98
+ metrics = _metrics(manifest)
99
+ found = []
100
+ for uid, node in sorted((manifest.get("nodes") or {}).items()):
101
+ config = node.get("config") or {}
102
+ if (
103
+ node.get("resource_type") != "model"
104
+ or config.get("enabled") is False
105
+ or config.get("materialized") not in MATERIALIZED
106
+ ):
107
+ continue
108
+ meta = _meta(node)
109
+ columns = [_column(c) for c in (node.get("columns") or {}).values() if isinstance(c, dict)]
110
+ described = {
111
+ **({"description": node["description"]} if node.get("description") else {}),
112
+ **{k: meta[k] for k in TABLE_META if k in meta},
113
+ **({"columns": columns} if columns else {}),
114
+ **({"metrics": metrics[uid]} if uid in metrics else {}),
115
+ }
116
+ path = _yml_path(root, node)
117
+ schemas.check({"$schema": schemas.TABLE, **described}, schemas.TABLE, path, report)
118
+ code = node.get("compiled_code")
119
+ sql_file = root / node.get("original_file_path", "")
120
+ guide = sql_file.with_suffix(".md")
121
+ found.append(
122
+ Table(
123
+ str(node.get("schema", "")),
124
+ str(node.get("alias") or node.get("name", "")),
125
+ record={
126
+ **described,
127
+ "materialized": config["materialized"],
128
+ **({"sql": code.strip()} if isinstance(code, str) and code.strip() else {}),
129
+ **({"source_path": node["original_file_path"]} if node.get("original_file_path") else {}),
130
+ },
131
+ path=path,
132
+ guide=guide.relative_to(root).as_posix() if guide.is_file() else None,
133
+ )
134
+ )
135
+ return found
136
+
137
+
138
+ _ENV_VAR = re.compile(r"""\{\{\s*env_var\(\s*(['"])(.+?)\1\s*(?:,\s*(['"])(.*?)\3\s*)?\)\s*\}\}""")
139
+
140
+
141
+ def _render(text: str, env: Mapping[str, str]) -> str | None:
142
+ """``text`` with `{{ env_var('NAME', 'default') }}` replaced. None when a value is missing."""
143
+ if any(m.group(2) not in env and m.group(4) is None for m in _ENV_VAR.finditer(text)):
144
+ return None
145
+ rendered = _ENV_VAR.sub(lambda m: env.get(m.group(2), m.group(4) or ""), text)
146
+ return None if "{" in rendered else rendered
147
+
148
+
149
+ _CATALOG_URL = re.compile(r"(https?://.+?/)(?:versions/[^/]+/)?(?:metadata\.db|ducklake\.duckdb)")
150
+
151
+
152
+ def _repository(path: str, env: Mapping[str, str]) -> str | None:
153
+ """The repository URL a DuckLake ATTACH path reads."""
154
+ rendered = _render(path, env)
155
+ match = _CATALOG_URL.fullmatch(rendered.removeprefix("ducklake:")) if rendered else None
156
+ return match.group(1) if match else None
157
+
158
+
159
+ def _target(profile: dict[str, Any], env: Mapping[str, str]) -> dict[str, Any] | None:
160
+ name = env.get("DBT_TARGET") or _render(str(profile.get("target") or "default"), env)
161
+ target = (profile.get("outputs") or {}).get(name)
162
+ return target if isinstance(target, dict) else None
163
+
164
+
165
+ def catalogs(profiles: dict[str, Any], env: Mapping[str, str] | None = None) -> dict[str, str]:
166
+ """Alias -> repository URL of each other dataset the profiles attach."""
167
+ env = os.environ if env is None else env
168
+ return {
169
+ str(attach["alias"]): repository
170
+ for name, profile in profiles.items()
171
+ if name != "config"
172
+ and isinstance(profile, dict)
173
+ and (target := _target(profile, env)) is not None
174
+ for attach in target.get("attach") or []
175
+ if isinstance(attach, dict)
176
+ and attach.get("alias")
177
+ and (repository := _repository(str(attach.get("path", "")), env)) is not None
178
+ }
179
+
180
+
181
+ def keys(found: list[Table]) -> set[Key]:
182
+ return {t.key for t in found}
@@ -0,0 +1,207 @@
1
+ """A dataset's description, read from what an author wrote under the root."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ import yaml
11
+
12
+ from . import dbt, files, git, lineage, shape, source
13
+ from .report import Report
14
+
15
+ README = "README.md"
16
+ DEFAULT_MANIFEST = Path("target") / "manifest.json"
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class Options:
21
+ root: Path
22
+ #: dbt's manifest.json; default target/manifest.json in a dbt project.
23
+ manifest: Path | None = None
24
+ #: dbt's profiles.yml, for the other datasets the SQL reads; default DBT_PROFILES_DIR or the root.
25
+ profiles: Path | None = None
26
+ #: Whether to say where it came from (`source`). Off for comparing outputs.
27
+ with_source: bool = True
28
+
29
+
30
+ @dataclass(frozen=True)
31
+ class Description:
32
+ """What the author wrote, with the older formats read as today's.
33
+
34
+ Records are plain dicts and lists, keys in a fixed order. Guides and the
35
+ files they link to are named by path; ``files`` lists each with its sha256,
36
+ and ``content`` gives what to send for one.
37
+ """
38
+
39
+ root: Path
40
+ dataset: dict[str, Any]
41
+ schemas: list[dict[str, Any]]
42
+ tables: list[dict[str, Any]]
43
+ files: list[files.File]
44
+ #: The Git working copy around the root; None outside Git or without ``with_source``.
45
+ source: dict[str, Any] | None = None
46
+ #: Alias -> repository URL of each other dataset the SQL reads (from dbt's profiles.yml).
47
+ catalogs: dict[str, str] | None = None
48
+
49
+ def content(self, file: files.File) -> bytes:
50
+ return files.content(self.root, file.path)
51
+
52
+
53
+ def _profiles(options: Options) -> dict[str, Any]:
54
+ import os
55
+
56
+ folder = os.environ.get("DBT_PROFILES_DIR")
57
+ path = options.profiles or (Path(folder) if folder else options.root) / "profiles.yml"
58
+ if not path.is_file():
59
+ return {}
60
+ loaded = yaml.safe_load(path.read_text(encoding="utf-8"))
61
+ return loaded if isinstance(loaded, dict) else {}
62
+
63
+
64
+ def _dbt_tables(options: Options, report: Report) -> tuple[list[source.Table], dict[str, str]]:
65
+ root = options.root
66
+ if not (root / "dbt_project.yml").is_file():
67
+ return [], {}
68
+ path = options.manifest or root / DEFAULT_MANIFEST
69
+ if not path.is_file():
70
+ report.warning(
71
+ "manifest-missing",
72
+ f"{path} is not here, so dbt's models are not read; run dbt compile first",
73
+ root / "dbt_project.yml",
74
+ )
75
+ return [], dbt.catalogs(_profiles(options))
76
+ manifest = json.loads(path.read_text(encoding="utf-8"))
77
+ return dbt.tables(manifest, root, report), dbt.catalogs(_profiles(options))
78
+
79
+
80
+ def _merge(
81
+ built: list[source.Table], written: list[source.Table], report: Report
82
+ ) -> list[source.Table]:
83
+ """One table per (schema, name). A table is described in one place only."""
84
+ by_key: dict[source.Key, source.Table] = {}
85
+ for table in written:
86
+ if table.key in by_key:
87
+ report.error(
88
+ "described-twice",
89
+ f"{table.schema}.{table.name} is also described in {by_key[table.key].path}",
90
+ table.path,
91
+ )
92
+ continue
93
+ by_key[table.key] = table
94
+ merged = dict(by_key)
95
+ for model in built:
96
+ found = by_key.get(model.key)
97
+ if found is None:
98
+ merged[model.key] = model
99
+ continue
100
+ if shape.described(model.record):
101
+ report.error(
102
+ "described-twice",
103
+ f"{model.schema}.{model.name} is described in dbt's yml and in {found.path}; keep one",
104
+ model.path,
105
+ )
106
+ built = {k: model.record[k] for k in shape.BUILT_KEYS if k in model.record and k not in found.record}
107
+ merged[model.key] = source.Table(
108
+ found.schema,
109
+ found.name,
110
+ record={**found.record, **built},
111
+ path=found.path,
112
+ guide=found.guide or model.guide,
113
+ )
114
+ return [merged[k] for k in sorted(merged)]
115
+
116
+
117
+ def _warn_metrics(tables: list[source.Table], report: Report) -> None:
118
+ for table in tables:
119
+ for metric in table.record.get("metrics") or []:
120
+ if isinstance(metric, dict) and not metric.get("agg"):
121
+ report.warning(
122
+ "metric-without-agg", f"{table.schema}.{table.name}: metric {metric.get('name')} has no agg", table.path
123
+ )
124
+
125
+
126
+ def _table(table: source.Table) -> dict[str, Any]:
127
+ shaped = shape.table(table.record)
128
+ head = {k: shaped[k] for k in shape.TABLE_KEYS if k in shaped}
129
+ tail = {k: v for k, v in shaped.items() if k not in shape.TABLE_KEYS}
130
+ return {
131
+ "schema": table.schema,
132
+ "name": table.name,
133
+ **head,
134
+ **({"guide": table.guide} if table.guide else {}),
135
+ **tail,
136
+ }
137
+
138
+
139
+ def _schemas(record: dict[str, Any]) -> list[dict[str, Any]]:
140
+ written = record.get("schemas")
141
+ return [
142
+ {**shape.pick(s, shape.SCHEMA_KEYS), **({"guide": files.normalized(s["guide"])} if s.get("guide") else {})}
143
+ for s in (written if isinstance(written, list) else [])
144
+ if isinstance(s, dict)
145
+ ]
146
+
147
+
148
+ def _documents(
149
+ dataset: source.Dataset, schema_list: list[dict[str, Any]], tables: list[source.Table], root: Path, report: Report
150
+ ) -> tuple[list[str], list[tuple[str, Path]]]:
151
+ """The dataset's docs as normalized paths, and every document named, with where it is named."""
152
+ written = dataset.record.get("docs")
153
+ docs = []
154
+ for entry in written if isinstance(written, list) else []:
155
+ path = files.normalized(str(entry))
156
+ if path is None:
157
+ report.error("file-outside", f"docs: {entry} is outside {root}", dataset.path)
158
+ else:
159
+ docs.append(path)
160
+ named = [
161
+ *([(README, root / README)] if (root / README).is_file() else []),
162
+ *[(s["guide"], dataset.path) for s in schema_list if s.get("guide")],
163
+ *[(d, dataset.path) for d in docs],
164
+ *[(t.guide, t.path) for t in tables if t.guide],
165
+ ]
166
+ for schema in schema_list:
167
+ if "guide" in schema and schema["guide"] is None:
168
+ report.error("file-outside", f"schemas: {schema['name']}'s guide is outside {root}", dataset.path)
169
+ return docs, [(p, w) for p, w in named if p]
170
+
171
+
172
+ def read(options: Options, report: Report) -> Description:
173
+ """Read and check the description under ``options.root``; findings go to ``report``."""
174
+ root = options.root.resolve()
175
+ options = Options(root, options.manifest, options.profiles, options.with_source)
176
+ dataset = source.read_dataset(root, report)
177
+ models, attached = _dbt_tables(options, report)
178
+ written = [*source.read_table_files(root, report), *source.read_legacy_tables(root, report)]
179
+ tables = _merge(models, written, report)
180
+ _warn_metrics(tables, report)
181
+ schema_list = _schemas(dataset.record)
182
+ docs, named = _documents(dataset, schema_list, tables, root, report)
183
+ sent = files.collect(root, named, report)
184
+ shaped = lineage.fill([_table(t) for t in tables], {**attached, **dataset.catalogs}, report)
185
+ record = dataset.record
186
+ return Description(
187
+ root=root,
188
+ dataset={
189
+ **shape.pick(record, shape.DATASET_KEYS),
190
+ **({"guide": README} if (root / README).is_file() else {}),
191
+ **({"docs": docs} if docs else {}),
192
+ **shape.annotations(record),
193
+ },
194
+ schemas=schema_list,
195
+ tables=[_ordered(t) for t in shaped],
196
+ files=sent,
197
+ source=git.source(root) if options.with_source else None,
198
+ catalogs={**attached, **dataset.catalogs} or None,
199
+ )
200
+
201
+
202
+ def _ordered(table: dict[str, Any]) -> dict[str, Any]:
203
+ """depends_on, filled in last, back in its place after sql."""
204
+ if "depends_on" not in table:
205
+ return table
206
+ order = ["schema", "name", *shape.TABLE_KEYS, "guide", "columns", "metrics", "annotations"]
207
+ return {k: table[k] for k in order if k in table}
tabledoc/files.py ADDED
@@ -0,0 +1,128 @@
1
+ """The files whose content is sent: guides, README.md, docs, and what they link to.
2
+
3
+ Each is named by its path under the root, with its sha256. A
4
+ Markdown table file of the older format is sent without its front matter.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import hashlib
10
+ import posixpath
11
+ import re
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from .report import Report
17
+ from .source import legacy_front_matter
18
+
19
+ #: Relative links in Markdown: [text](target), ![alt](target), and src="target".
20
+ _LINK = re.compile(r"""(\]\(\s*<?)([^)\s>]+)|(\bsrc\s*=\s*["'])([^"']+)""")
21
+
22
+ #: Fixed, so the same file gets the same type on every machine.
23
+ MEDIA_TYPES = {
24
+ ".md": "text/markdown",
25
+ ".markdown": "text/markdown",
26
+ ".txt": "text/plain",
27
+ ".csv": "text/csv",
28
+ ".tsv": "text/tab-separated-values",
29
+ ".html": "text/html",
30
+ ".htm": "text/html",
31
+ ".json": "application/json",
32
+ ".geojson": "application/geo+json",
33
+ ".yml": "application/yaml",
34
+ ".yaml": "application/yaml",
35
+ ".xml": "application/xml",
36
+ ".pdf": "application/pdf",
37
+ ".zip": "application/zip",
38
+ ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
39
+ ".xls": "application/vnd.ms-excel",
40
+ ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
41
+ ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
42
+ ".png": "image/png",
43
+ ".jpg": "image/jpeg",
44
+ ".jpeg": "image/jpeg",
45
+ ".gif": "image/gif",
46
+ ".svg": "image/svg+xml",
47
+ ".webp": "image/webp",
48
+ ".avif": "image/avif",
49
+ }
50
+ OCTET_STREAM = "application/octet-stream"
51
+
52
+
53
+ def media_type(path: str) -> str:
54
+ return MEDIA_TYPES.get(posixpath.splitext(path)[1].lower(), OCTET_STREAM)
55
+
56
+
57
+ def content(root: Path, path: str) -> bytes:
58
+ """What is sent for ``path``: the file, or a Markdown table file's text below its front matter."""
59
+ file = root / path
60
+ if path.lower().endswith(".md"):
61
+ found = legacy_front_matter(file)
62
+ if found is not None:
63
+ return found[1].encode("utf-8")
64
+ return file.read_bytes()
65
+
66
+
67
+ def _is_relative(target: str) -> bool:
68
+ return not (target.startswith(("#", "/")) or re.match(r"^[a-z][a-z0-9+.-]*:", target, re.IGNORECASE))
69
+
70
+
71
+ def links(text: str) -> list[str]:
72
+ """The relative link targets in ``text``, without #fragment or ?query."""
73
+ found = (m.group(2) or m.group(4) for m in _LINK.finditer(text))
74
+ return [t for t in (re.split(r"[?#]", f)[0] for f in found if _is_relative(f)) if t]
75
+
76
+
77
+ def normalized(path: str) -> str | None:
78
+ """``path`` normalized, or None when it leaves the root."""
79
+ clean = posixpath.normpath(path)
80
+ return None if clean == ".." or clean.startswith("../") or posixpath.isabs(clean) else clean
81
+
82
+
83
+ @dataclass(frozen=True)
84
+ class File:
85
+ path: str
86
+ sha256: str
87
+ bytes: int
88
+ media_type: str
89
+
90
+ def entry(self) -> dict[str, Any]:
91
+ return {"path": self.path, "sha256": self.sha256, "bytes": self.bytes, "media_type": self.media_type}
92
+
93
+
94
+ def describe(root: Path, path: str) -> File:
95
+ body = content(root, path)
96
+ return File(path, hashlib.sha256(body).hexdigest(), len(body), media_type(path))
97
+
98
+
99
+ def collect(root: Path, documents: list[tuple[str, Path]], report: Report) -> list[File]:
100
+ """``documents`` (path, the file that names it) and the files their Markdown links to.
101
+
102
+ A named document that is missing, or outside the root, is an error. A link
103
+ to a folder is left alone; a link to a missing file is a warning.
104
+ """
105
+ seen: dict[str, File] = {}
106
+ pending = list(documents)
107
+ named = {path for path, _ in documents}
108
+ while pending:
109
+ path, where = pending.pop(0)
110
+ if path in seen:
111
+ continue
112
+ file = root / path
113
+ if not file.is_file() or not file.resolve().is_relative_to(root.resolve()):
114
+ if path in named:
115
+ report.error("file-missing", f"{path} is not a file under {root}", where)
116
+ elif not file.is_dir():
117
+ report.warning("link-broken", f"links to {path}, which is not here", where)
118
+ continue
119
+ seen[path] = describe(root, path)
120
+ if path.lower().endswith(".md"):
121
+ text = content(root, path).decode("utf-8", errors="replace")
122
+ for target in links(text):
123
+ linked = normalized(posixpath.join(posixpath.dirname(path), target))
124
+ if linked is None:
125
+ report.error("link-outside", f"links to {target}, outside {root}", file)
126
+ elif linked not in seen:
127
+ pending.append((linked, file))
128
+ return [seen[p] for p in sorted(seen)]
tabledoc/git.py ADDED
@@ -0,0 +1,43 @@
1
+ """Where a description came from: the Git working copy around the root."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ import subprocess
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+
11
+ def _git(root: Path, *args: str) -> str | None:
12
+ try:
13
+ done = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True, check=False)
14
+ except FileNotFoundError:
15
+ return None
16
+ return done.stdout.strip() if done.returncode == 0 else None
17
+
18
+
19
+ def https(url: str) -> str:
20
+ """A remote URL as https, without credentials or `.git`: git@github.com:a/b.git -> https://github.com/a/b."""
21
+ scp = re.fullmatch(r"(?:[^@/]+@)?([^:/]+):(?!//)(.+)", url)
22
+ if scp:
23
+ url = f"https://{scp.group(1)}/{scp.group(2)}"
24
+ url = re.sub(r"^(?:ssh|git|http)://", "https://", url)
25
+ url = re.sub(r"^https://[^@/]+@", "https://", url)
26
+ return url.removesuffix("/").removesuffix(".git")
27
+
28
+
29
+ def source(root: Path) -> dict[str, Any] | None:
30
+ """{repository, commit, path, dirty}, or None outside Git."""
31
+ top = _git(root, "rev-parse", "--show-toplevel")
32
+ commit = _git(root, "rev-parse", "HEAD")
33
+ if top is None or commit is None:
34
+ return None
35
+ remote = _git(root, "remote", "get-url", "origin")
36
+ status = _git(root, "status", "--porcelain")
37
+ path = root.resolve().relative_to(Path(top).resolve()).as_posix()
38
+ return {
39
+ **({"repository": https(remote)} if remote else {}),
40
+ "commit": commit,
41
+ "path": "" if path == "." else path,
42
+ "dirty": bool(status),
43
+ }