xref 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xref/__init__.py ADDED
@@ -0,0 +1 @@
1
+
xref/_typing.py ADDED
@@ -0,0 +1,13 @@
1
+ """Narrow untyped JSON and YAML values to concrete types."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, cast
6
+
7
+
8
+ def as_mapping(value: object) -> dict[str, Any] | None:
9
+ return cast(dict[str, Any], value) if isinstance(value, dict) else None
10
+
11
+
12
+ def as_list(value: object) -> list[Any] | None:
13
+ return cast(list[Any], value) if isinstance(value, list) else None
xref/cli.py ADDED
@@ -0,0 +1,138 @@
1
+ """xref command line."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ from importlib.metadata import version as package_version
7
+ from pathlib import Path
8
+ from typing import Annotated
9
+
10
+ import typer
11
+
12
+ from xref.config import DEFAULT_ENVIRONMENT, load_config
13
+ from xref.errors import XrefError
14
+ from xref.operations import age_of, publish, sync
15
+ from xref.store import open_store
16
+
17
+ app = typer.Typer(
18
+ help="Cross-project refs for dbt v2 without the dbt platform.",
19
+ no_args_is_help=True,
20
+ add_completion=False,
21
+ )
22
+
23
+ ProjectDir = Annotated[
24
+ Path,
25
+ typer.Option("--project-dir", help="The dbt project directory.", file_okay=False, exists=True),
26
+ ]
27
+ Environment = Annotated[
28
+ str | None,
29
+ typer.Option(
30
+ "--env",
31
+ "-e",
32
+ help=f"Environment name. Defaults to $XREF_ENV, then {DEFAULT_ENVIRONMENT!r}.",
33
+ ),
34
+ ]
35
+ StoreUrl = Annotated[
36
+ str | None,
37
+ typer.Option(
38
+ "--store",
39
+ help="Store URL or directory. Defaults to $XREF_STORE, then `store` in xref.yml.",
40
+ ),
41
+ ]
42
+
43
+
44
+ def _version_callback(value: bool) -> None:
45
+ if value:
46
+ typer.echo(f"xref {package_version('xref')}")
47
+ raise typer.Exit()
48
+
49
+
50
+ @app.callback()
51
+ def main(
52
+ version: Annotated[
53
+ bool,
54
+ typer.Option(
55
+ "--version",
56
+ callback=_version_callback,
57
+ is_eager=True,
58
+ help="Show the version.",
59
+ ),
60
+ ] = False,
61
+ ) -> None:
62
+ """Publish a producer's public models; sync them into consumers as stub packages."""
63
+
64
+
65
+ def _environment(env: str | None) -> str:
66
+ return env or os.environ.get("XREF_ENV") or DEFAULT_ENVIRONMENT
67
+
68
+
69
+ def _store_url(store: str | None, configured: str | None) -> str:
70
+ url = store or os.environ.get("XREF_STORE") or configured
71
+ if not url:
72
+ raise XrefError(
73
+ "No store configured. Pass --store, set XREF_STORE, or set `store:` in xref.yml."
74
+ )
75
+ return url
76
+
77
+
78
+ def _fail(error: XrefError) -> typer.Exit:
79
+ typer.secho(f"Error: {error}", fg=typer.colors.RED, err=True)
80
+ return typer.Exit(code=1)
81
+
82
+
83
+ @app.command("publish")
84
+ def publish_command(
85
+ project_dir: ProjectDir = Path("."),
86
+ env: Environment = None,
87
+ store: StoreUrl = None,
88
+ manifest: Annotated[
89
+ Path | None,
90
+ typer.Option(help="Path to manifest.json. Defaults to <project-dir>/target/manifest.json."),
91
+ ] = None,
92
+ ) -> None:
93
+ """Publish this project's public models after a dbt run."""
94
+ try:
95
+ config = load_config(project_dir)
96
+ target = open_store(_store_url(store, config.store), base_dir=project_dir)
97
+ result = publish(
98
+ manifest or project_dir / "target" / "manifest.json",
99
+ target,
100
+ _environment(env),
101
+ )
102
+ except XrefError as error:
103
+ raise _fail(error) from None
104
+ for warning in result.warnings:
105
+ typer.secho(f"Warning: {warning}", fg=typer.colors.YELLOW, err=True)
106
+ published = result.published
107
+ typer.echo(
108
+ f"Published {len(published.models)} public model(s) from {published.project_name} "
109
+ f"({published.environment}) to {result.location}"
110
+ )
111
+
112
+
113
+ @app.command("sync")
114
+ def sync_command(
115
+ project_dir: ProjectDir = Path("."),
116
+ env: Environment = None,
117
+ store: StoreUrl = None,
118
+ ) -> None:
119
+ """Fetch upstream manifests and write stub packages. Run before `dbt deps`."""
120
+ try:
121
+ config = load_config(project_dir)
122
+ source = open_store(_store_url(store, config.store), base_dir=project_dir)
123
+ result = sync(project_dir, config, source, _environment(env))
124
+ except XrefError as error:
125
+ raise _fail(error) from None
126
+ for project in result.projects:
127
+ published = project.published
128
+ typer.echo(
129
+ f"Synced {published.project_name} ({published.environment}, built "
130
+ f"{age_of(published.generated_at)}): {len(published.models)} model(s) -> "
131
+ f"{project.package_dir.relative_to(project_dir)}"
132
+ )
133
+ for name in result.removed:
134
+ typer.echo(f"Removed stub package for {name}, which is no longer in xref.yml")
135
+ if result.packages_added and result.packages_file is not None:
136
+ typer.echo(
137
+ f"Added {', '.join(result.packages_added)} to {result.packages_file.name}; run `dbt deps` next."
138
+ )
xref/config.py ADDED
@@ -0,0 +1,88 @@
1
+ """xref.yml: per-project settings, read from the dbt project directory.
2
+
3
+ store: s3://analytics-artifacts/xref # where manifests are published
4
+ projects: # upstream projects this one refs
5
+ - core_platform
6
+ - name: finance
7
+ environment: prod # always read this environment
8
+ environments: # which upstream environments each
9
+ dev: [staging, prod] # local environment reads, in order
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from pathlib import Path, PurePosixPath
15
+ from typing import Any
16
+
17
+ import yaml
18
+ from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
19
+
20
+ from xref._typing import as_list
21
+ from xref.errors import XrefError
22
+ from xref.manifest import PROJECT_NAME
23
+
24
+ CONFIG_FILE = "xref.yml"
25
+ DEFAULT_ENVIRONMENT = "prod"
26
+
27
+
28
+ class Upstream(BaseModel):
29
+ model_config = ConfigDict(extra="forbid")
30
+
31
+ name: str
32
+ environment: str | None = None
33
+
34
+ @field_validator("name")
35
+ @classmethod
36
+ def _valid_name(cls, value: str) -> str:
37
+ if not PROJECT_NAME.match(value):
38
+ raise ValueError(f"{value!r} is not a valid dbt project name")
39
+ return value
40
+
41
+
42
+ class Config(BaseModel):
43
+ model_config = ConfigDict(extra="forbid")
44
+
45
+ store: str | None = None
46
+ projects: list[Upstream] = []
47
+ environments: dict[str, list[str]] = {}
48
+ mesh_dir: str = ".xref"
49
+
50
+ @field_validator("projects", mode="before")
51
+ @classmethod
52
+ def _names_as_strings(cls, value: Any) -> Any:
53
+ items = as_list(value)
54
+ if items is None:
55
+ return value
56
+ return [{"name": v} if isinstance(v, str) else v for v in items]
57
+
58
+ @field_validator("mesh_dir")
59
+ @classmethod
60
+ def _mesh_dir_inside_project(cls, value: str) -> str:
61
+ path = PurePosixPath(value)
62
+ if path.is_absolute() or ".." in path.parts or not path.parts:
63
+ raise ValueError("mesh_dir must be a relative path inside the project, like .xref")
64
+ return value
65
+
66
+ def environments_for(self, upstream: Upstream, environment: str) -> list[str]:
67
+ """The upstream environments to try, in order, when running in `environment`."""
68
+ if upstream.environment is not None:
69
+ return [upstream.environment]
70
+ if environment in self.environments:
71
+ return self.environments[environment]
72
+ if environment == DEFAULT_ENVIRONMENT:
73
+ return [DEFAULT_ENVIRONMENT]
74
+ return [environment, DEFAULT_ENVIRONMENT]
75
+
76
+
77
+ def load_config(project_dir: Path) -> Config:
78
+ path = project_dir / CONFIG_FILE
79
+ if not path.exists():
80
+ return Config()
81
+ try:
82
+ raw: Any = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
83
+ except yaml.YAMLError as exc:
84
+ raise XrefError(f"{path} is not valid YAML: {exc}") from None
85
+ try:
86
+ return Config.model_validate(raw)
87
+ except ValidationError as exc:
88
+ raise XrefError(f"{path} is not a valid xref config:\n{exc}") from None
xref/errors.py ADDED
@@ -0,0 +1,2 @@
1
+ class XrefError(Exception):
2
+ """An error with a message meant for the person running xref."""
xref/manifest.py ADDED
@@ -0,0 +1,128 @@
1
+ """Read the parts of a dbt manifest.json that xref needs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ from pathlib import Path
8
+ from typing import Any, Literal
9
+
10
+ from pydantic import BaseModel, ConfigDict, Field, ValidationError
11
+
12
+ from xref._typing import as_mapping
13
+ from xref.errors import XrefError
14
+
15
+ SUPPORTED_SCHEMA_VERSIONS = frozenset({"https://schemas.getdbt.com/dbt/manifest/v12.json"})
16
+
17
+ PROJECT_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
18
+
19
+ Access = Literal["public", "protected", "private"]
20
+ ModelVersion = int | float | str
21
+
22
+
23
+ class Column(BaseModel):
24
+ model_config = ConfigDict(extra="ignore")
25
+
26
+ name: str
27
+ description: str = ""
28
+ data_type: str | None = None
29
+
30
+
31
+ class ModelNode(BaseModel):
32
+ model_config = ConfigDict(extra="ignore", populate_by_name=True)
33
+
34
+ unique_id: str
35
+ name: str
36
+ package_name: str
37
+ access: Access = "protected"
38
+ database: str | None = None
39
+ schema_name: str = Field(alias="schema")
40
+ alias: str | None = None
41
+ relation_name: str | None = None
42
+ description: str = ""
43
+ columns: dict[str, Column] = Field(default_factory=dict[str, Column])
44
+ version: ModelVersion | None = None
45
+ latest_version: ModelVersion | None = None
46
+ deprecation_date: str | None = None
47
+
48
+
49
+ class Metadata(BaseModel):
50
+ model_config = ConfigDict(extra="ignore")
51
+
52
+ dbt_schema_version: str
53
+ dbt_version: str
54
+ generated_at: str
55
+ project_name: str | None = None
56
+ adapter_type: str | None = None
57
+
58
+
59
+ class Manifest(BaseModel):
60
+ """A dbt manifest reduced to the project's own models."""
61
+
62
+ metadata: Metadata
63
+ project_name: str
64
+ models: list[ModelNode]
65
+
66
+ def public_models(self) -> list[ModelNode]:
67
+ return [m for m in self.models if m.access == "public"]
68
+
69
+
70
+ def load_manifest(path: Path) -> Manifest:
71
+ try:
72
+ raw = as_mapping(json.loads(path.read_text(encoding="utf-8")))
73
+ except FileNotFoundError:
74
+ raise XrefError(f"No manifest at {path}. Run `dbt parse` or `dbt build` first.") from None
75
+ except json.JSONDecodeError as exc:
76
+ raise XrefError(f"{path} is not valid JSON: {exc}") from None
77
+ if raw is None:
78
+ raise XrefError(f"{path} is not a dbt manifest.")
79
+ return parse_manifest(raw, source=str(path))
80
+
81
+
82
+ def parse_manifest(raw: dict[str, Any], source: str = "manifest") -> Manifest:
83
+ try:
84
+ metadata = Metadata.model_validate(raw.get("metadata"))
85
+ except ValidationError as exc:
86
+ raise XrefError(f"{source} has no usable metadata block: {exc}") from None
87
+
88
+ if metadata.dbt_schema_version not in SUPPORTED_SCHEMA_VERSIONS:
89
+ supported = ", ".join(sorted(SUPPORTED_SCHEMA_VERSIONS))
90
+ raise XrefError(
91
+ f"{source} uses manifest schema {metadata.dbt_schema_version}; xref supports {supported}."
92
+ )
93
+
94
+ nodes = as_mapping(raw.get("nodes", {}))
95
+ if nodes is None:
96
+ raise XrefError(f"{source} has no nodes.")
97
+
98
+ project_name = metadata.project_name or _infer_project_name(nodes)
99
+ if not PROJECT_NAME.match(project_name):
100
+ raise XrefError(f"{source}: project name {project_name!r} is not a valid dbt project name.")
101
+
102
+ models: list[ModelNode] = []
103
+ for unique_id, value in nodes.items():
104
+ node = as_mapping(value)
105
+ if node is None or node.get("resource_type") != "model":
106
+ continue
107
+ # Models from installed packages (dbt_utils and so on) belong to other projects.
108
+ if node.get("package_name") != project_name:
109
+ continue
110
+ try:
111
+ models.append(ModelNode.model_validate(node))
112
+ except ValidationError as exc:
113
+ raise XrefError(f"{source}: could not read {unique_id}: {exc}") from None
114
+
115
+ models.sort(key=lambda m: m.unique_id)
116
+ return Manifest(metadata=metadata, project_name=project_name, models=models)
117
+
118
+
119
+ def _infer_project_name(nodes: dict[str, Any]) -> str:
120
+ """Older manifests lack metadata.project_name; fall back to the root project's nodes."""
121
+ packages = {
122
+ str(node.get("package_name"))
123
+ for node in map(as_mapping, nodes.values())
124
+ if node is not None and node.get("resource_type") == "model"
125
+ }
126
+ if len(packages) == 1:
127
+ return packages.pop()
128
+ raise XrefError("The manifest has no metadata.project_name and xref could not infer it.")
xref/operations.py ADDED
@@ -0,0 +1,196 @@
1
+ """The two operations behind the CLI: publish and sync."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import shutil
6
+ import tempfile
7
+ from dataclasses import dataclass, field
8
+ from datetime import UTC, datetime
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import yaml
13
+
14
+ from xref._typing import as_mapping
15
+ from xref.config import Config
16
+ from xref.errors import XrefError
17
+ from xref.manifest import load_manifest
18
+ from xref.packages import ensure_local_packages, remove_local_packages
19
+ from xref.published import PublishedManifest, from_manifest, parse_published
20
+ from xref.store import ManifestStore, manifest_key
21
+ from xref.stubs import render_stub_package
22
+
23
+ MARKER_FILE = ".xref.json"
24
+
25
+
26
+ @dataclass
27
+ class PublishResult:
28
+ published: PublishedManifest
29
+ location: str
30
+ warnings: list[str]
31
+
32
+
33
+ def publish(
34
+ manifest_path: Path,
35
+ store: ManifestStore,
36
+ environment: str,
37
+ now: datetime | None = None,
38
+ ) -> PublishResult:
39
+ manifest = load_manifest(manifest_path)
40
+ published, warnings = from_manifest(manifest, environment, now)
41
+ key = manifest_key(published.project_name, environment)
42
+ store.write(key, published.model_dump_json(indent=2).encode())
43
+ return PublishResult(published=published, location=store.describe(key), warnings=warnings)
44
+
45
+
46
+ @dataclass
47
+ class SyncedProject:
48
+ published: PublishedManifest
49
+ source: str
50
+ package_dir: Path
51
+
52
+
53
+ @dataclass
54
+ class SyncResult:
55
+ projects: list[SyncedProject]
56
+ packages_file: Path | None
57
+ packages_added: list[str]
58
+ removed: list[str] = field(default_factory=list[str])
59
+
60
+
61
+ def sync(project_dir: Path, config: Config, store: ManifestStore, environment: str) -> SyncResult:
62
+ if not config.projects:
63
+ raise XrefError(
64
+ f"No upstream projects listed under `projects:` in {project_dir / 'xref.yml'}."
65
+ )
66
+
67
+ own_name = _own_project_name(project_dir)
68
+ mesh_dir = project_dir / config.mesh_dir
69
+ synced: list[SyncedProject] = []
70
+
71
+ # Fetch every upstream before writing anything, so one missing manifest
72
+ # does not leave some packages updated and others stale.
73
+ for upstream in config.projects:
74
+ if upstream.name == own_name:
75
+ raise XrefError(
76
+ f"{upstream.name} is this project's own name; it cannot be its own upstream."
77
+ )
78
+ published, source = _fetch(
79
+ store, upstream.name, config.environments_for(upstream, environment)
80
+ )
81
+ if published.project_name != upstream.name:
82
+ raise XrefError(
83
+ f"{source} holds project {published.project_name!r}, expected {upstream.name!r}."
84
+ )
85
+ synced.append(
86
+ SyncedProject(published=published, source=source, package_dir=mesh_dir / upstream.name)
87
+ )
88
+
89
+ for project in synced:
90
+ _write_package(project.package_dir, project.published, project.source)
91
+
92
+ removed = _remove_stale(project_dir, mesh_dir, {u.name for u in config.projects})
93
+ rel_paths = [p.package_dir.relative_to(project_dir).as_posix() for p in synced]
94
+ packages_path, added = ensure_local_packages(project_dir, rel_paths)
95
+ return SyncResult(
96
+ projects=synced,
97
+ packages_file=packages_path,
98
+ packages_added=added,
99
+ removed=removed,
100
+ )
101
+
102
+
103
+ def _fetch(
104
+ store: ManifestStore, project: str, environments: list[str]
105
+ ) -> tuple[PublishedManifest, str]:
106
+ tried: list[str] = []
107
+ for env in environments:
108
+ key = manifest_key(project, env)
109
+ data = store.read(key)
110
+ location = store.describe(key)
111
+ if data is not None:
112
+ return parse_published(data, location), location
113
+ tried.append(location)
114
+ raise XrefError(
115
+ f"No published manifest for {project}. Tried:\n"
116
+ + "\n".join(f" {t}" for t in tried)
117
+ + f"\nHas {project} run `xref publish`?"
118
+ )
119
+
120
+
121
+ def _write_package(package_dir: Path, published: PublishedManifest, source: str) -> None:
122
+ if package_dir.exists() and not (package_dir / MARKER_FILE).exists():
123
+ raise XrefError(
124
+ f"{package_dir} exists and was not created by xref; move it before syncing."
125
+ )
126
+ package_dir.parent.mkdir(parents=True, exist_ok=True)
127
+ # Build the new package beside the old one, then swap, so a failed sync
128
+ # never leaves a half-written package behind.
129
+ staging = Path(tempfile.mkdtemp(dir=package_dir.parent, prefix=f".{package_dir.name}-"))
130
+ try:
131
+ for rel, content in render_stub_package(published).items():
132
+ target = staging / rel
133
+ target.parent.mkdir(parents=True, exist_ok=True)
134
+ target.write_text(content, encoding="utf-8")
135
+ marker: dict[str, Any] = {
136
+ "source": source,
137
+ "environment": published.environment,
138
+ "generated_at": published.generated_at,
139
+ "published_at": published.published_at,
140
+ }
141
+ (staging / MARKER_FILE).write_text(
142
+ yaml.safe_dump(marker, sort_keys=False), encoding="utf-8"
143
+ )
144
+ if package_dir.exists():
145
+ shutil.rmtree(package_dir)
146
+ staging.rename(package_dir)
147
+ except BaseException:
148
+ shutil.rmtree(staging, ignore_errors=True)
149
+ raise
150
+
151
+
152
+ def _remove_stale(project_dir: Path, mesh_dir: Path, keep: set[str]) -> list[str]:
153
+ """Delete stub packages for projects no longer in xref.yml, and their packages entries."""
154
+ if not mesh_dir.exists():
155
+ return []
156
+ stale = [
157
+ child
158
+ for child in sorted(mesh_dir.iterdir())
159
+ if child.is_dir() and child.name not in keep and (child / MARKER_FILE).exists()
160
+ ]
161
+ # Remove the packages entries first: if that fails, the folders stay and dbt deps still works.
162
+ remove_local_packages(project_dir, [c.relative_to(project_dir).as_posix() for c in stale])
163
+ for child in stale:
164
+ shutil.rmtree(child)
165
+ return [c.name for c in stale]
166
+
167
+
168
+ def _own_project_name(project_dir: Path) -> str | None:
169
+ path = project_dir / "dbt_project.yml"
170
+ if not path.exists():
171
+ raise XrefError(
172
+ f"No dbt_project.yml in {project_dir}. Run xref from the dbt project directory or pass --project-dir."
173
+ )
174
+ try:
175
+ data = as_mapping(yaml.safe_load(path.read_text(encoding="utf-8")) or {})
176
+ except yaml.YAMLError as exc:
177
+ raise XrefError(f"{path} is not valid YAML: {exc}") from None
178
+ name = data.get("name") if data is not None else None
179
+ return name if isinstance(name, str) else None
180
+
181
+
182
+ def age_of(timestamp: str, now: datetime | None = None) -> str:
183
+ """'3h ago' style age for a manifest timestamp, or the raw value if unparseable."""
184
+ try:
185
+ then = datetime.fromisoformat(timestamp)
186
+ except ValueError:
187
+ return timestamp
188
+ if then.tzinfo is None:
189
+ then = then.replace(tzinfo=UTC)
190
+ seconds = int(((now or datetime.now(UTC)) - then).total_seconds())
191
+ if seconds < 0:
192
+ return "just now"
193
+ for unit, size in (("d", 86400), ("h", 3600), ("m", 60)):
194
+ if seconds >= size:
195
+ return f"{seconds // size}{unit} ago"
196
+ return f"{seconds}s ago"