xref 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xref/__init__.py +1 -0
- xref/_typing.py +13 -0
- xref/cli.py +138 -0
- xref/config.py +88 -0
- xref/errors.py +2 -0
- xref/manifest.py +128 -0
- xref/operations.py +196 -0
- xref/packages.py +160 -0
- xref/published.py +126 -0
- xref/store.py +139 -0
- xref/stubs.py +108 -0
- xref-0.1.0.dist-info/METADATA +181 -0
- xref-0.1.0.dist-info/RECORD +16 -0
- xref-0.1.0.dist-info/WHEEL +4 -0
- xref-0.1.0.dist-info/entry_points.txt +3 -0
- xref-0.1.0.dist-info/licenses/LICENSE +177 -0
xref/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
xref/_typing.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Narrow untyped JSON and YAML values to concrete types."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, cast
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def as_mapping(value: object) -> dict[str, Any] | None:
|
|
9
|
+
return cast(dict[str, Any], value) if isinstance(value, dict) else None
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def as_list(value: object) -> list[Any] | None:
|
|
13
|
+
return cast(list[Any], value) if isinstance(value, list) else None
|
xref/cli.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""xref command line."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from importlib.metadata import version as package_version
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Annotated
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
|
|
12
|
+
from xref.config import DEFAULT_ENVIRONMENT, load_config
|
|
13
|
+
from xref.errors import XrefError
|
|
14
|
+
from xref.operations import age_of, publish, sync
|
|
15
|
+
from xref.store import open_store
|
|
16
|
+
|
|
17
|
+
app = typer.Typer(
|
|
18
|
+
help="Cross-project refs for dbt v2 without the dbt platform.",
|
|
19
|
+
no_args_is_help=True,
|
|
20
|
+
add_completion=False,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
ProjectDir = Annotated[
|
|
24
|
+
Path,
|
|
25
|
+
typer.Option("--project-dir", help="The dbt project directory.", file_okay=False, exists=True),
|
|
26
|
+
]
|
|
27
|
+
Environment = Annotated[
|
|
28
|
+
str | None,
|
|
29
|
+
typer.Option(
|
|
30
|
+
"--env",
|
|
31
|
+
"-e",
|
|
32
|
+
help=f"Environment name. Defaults to $XREF_ENV, then {DEFAULT_ENVIRONMENT!r}.",
|
|
33
|
+
),
|
|
34
|
+
]
|
|
35
|
+
StoreUrl = Annotated[
|
|
36
|
+
str | None,
|
|
37
|
+
typer.Option(
|
|
38
|
+
"--store",
|
|
39
|
+
help="Store URL or directory. Defaults to $XREF_STORE, then `store` in xref.yml.",
|
|
40
|
+
),
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _version_callback(value: bool) -> None:
|
|
45
|
+
if value:
|
|
46
|
+
typer.echo(f"xref {package_version('xref')}")
|
|
47
|
+
raise typer.Exit()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@app.callback()
|
|
51
|
+
def main(
|
|
52
|
+
version: Annotated[
|
|
53
|
+
bool,
|
|
54
|
+
typer.Option(
|
|
55
|
+
"--version",
|
|
56
|
+
callback=_version_callback,
|
|
57
|
+
is_eager=True,
|
|
58
|
+
help="Show the version.",
|
|
59
|
+
),
|
|
60
|
+
] = False,
|
|
61
|
+
) -> None:
|
|
62
|
+
"""Publish a producer's public models; sync them into consumers as stub packages."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _environment(env: str | None) -> str:
|
|
66
|
+
return env or os.environ.get("XREF_ENV") or DEFAULT_ENVIRONMENT
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _store_url(store: str | None, configured: str | None) -> str:
|
|
70
|
+
url = store or os.environ.get("XREF_STORE") or configured
|
|
71
|
+
if not url:
|
|
72
|
+
raise XrefError(
|
|
73
|
+
"No store configured. Pass --store, set XREF_STORE, or set `store:` in xref.yml."
|
|
74
|
+
)
|
|
75
|
+
return url
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _fail(error: XrefError) -> typer.Exit:
|
|
79
|
+
typer.secho(f"Error: {error}", fg=typer.colors.RED, err=True)
|
|
80
|
+
return typer.Exit(code=1)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@app.command("publish")
|
|
84
|
+
def publish_command(
|
|
85
|
+
project_dir: ProjectDir = Path("."),
|
|
86
|
+
env: Environment = None,
|
|
87
|
+
store: StoreUrl = None,
|
|
88
|
+
manifest: Annotated[
|
|
89
|
+
Path | None,
|
|
90
|
+
typer.Option(help="Path to manifest.json. Defaults to <project-dir>/target/manifest.json."),
|
|
91
|
+
] = None,
|
|
92
|
+
) -> None:
|
|
93
|
+
"""Publish this project's public models after a dbt run."""
|
|
94
|
+
try:
|
|
95
|
+
config = load_config(project_dir)
|
|
96
|
+
target = open_store(_store_url(store, config.store), base_dir=project_dir)
|
|
97
|
+
result = publish(
|
|
98
|
+
manifest or project_dir / "target" / "manifest.json",
|
|
99
|
+
target,
|
|
100
|
+
_environment(env),
|
|
101
|
+
)
|
|
102
|
+
except XrefError as error:
|
|
103
|
+
raise _fail(error) from None
|
|
104
|
+
for warning in result.warnings:
|
|
105
|
+
typer.secho(f"Warning: {warning}", fg=typer.colors.YELLOW, err=True)
|
|
106
|
+
published = result.published
|
|
107
|
+
typer.echo(
|
|
108
|
+
f"Published {len(published.models)} public model(s) from {published.project_name} "
|
|
109
|
+
f"({published.environment}) to {result.location}"
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@app.command("sync")
|
|
114
|
+
def sync_command(
|
|
115
|
+
project_dir: ProjectDir = Path("."),
|
|
116
|
+
env: Environment = None,
|
|
117
|
+
store: StoreUrl = None,
|
|
118
|
+
) -> None:
|
|
119
|
+
"""Fetch upstream manifests and write stub packages. Run before `dbt deps`."""
|
|
120
|
+
try:
|
|
121
|
+
config = load_config(project_dir)
|
|
122
|
+
source = open_store(_store_url(store, config.store), base_dir=project_dir)
|
|
123
|
+
result = sync(project_dir, config, source, _environment(env))
|
|
124
|
+
except XrefError as error:
|
|
125
|
+
raise _fail(error) from None
|
|
126
|
+
for project in result.projects:
|
|
127
|
+
published = project.published
|
|
128
|
+
typer.echo(
|
|
129
|
+
f"Synced {published.project_name} ({published.environment}, built "
|
|
130
|
+
f"{age_of(published.generated_at)}): {len(published.models)} model(s) -> "
|
|
131
|
+
f"{project.package_dir.relative_to(project_dir)}"
|
|
132
|
+
)
|
|
133
|
+
for name in result.removed:
|
|
134
|
+
typer.echo(f"Removed stub package for {name}, which is no longer in xref.yml")
|
|
135
|
+
if result.packages_added and result.packages_file is not None:
|
|
136
|
+
typer.echo(
|
|
137
|
+
f"Added {', '.join(result.packages_added)} to {result.packages_file.name}; run `dbt deps` next."
|
|
138
|
+
)
|
xref/config.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""xref.yml: per-project settings, read from the dbt project directory.
|
|
2
|
+
|
|
3
|
+
store: s3://analytics-artifacts/xref # where manifests are published
|
|
4
|
+
projects: # upstream projects this one refs
|
|
5
|
+
- core_platform
|
|
6
|
+
- name: finance
|
|
7
|
+
environment: prod # always read this environment
|
|
8
|
+
environments: # which upstream environments each
|
|
9
|
+
dev: [staging, prod] # local environment reads, in order
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from pathlib import Path, PurePosixPath
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import yaml
|
|
18
|
+
from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
|
|
19
|
+
|
|
20
|
+
from xref._typing import as_list
|
|
21
|
+
from xref.errors import XrefError
|
|
22
|
+
from xref.manifest import PROJECT_NAME
|
|
23
|
+
|
|
24
|
+
CONFIG_FILE = "xref.yml"
|
|
25
|
+
DEFAULT_ENVIRONMENT = "prod"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Upstream(BaseModel):
|
|
29
|
+
model_config = ConfigDict(extra="forbid")
|
|
30
|
+
|
|
31
|
+
name: str
|
|
32
|
+
environment: str | None = None
|
|
33
|
+
|
|
34
|
+
@field_validator("name")
|
|
35
|
+
@classmethod
|
|
36
|
+
def _valid_name(cls, value: str) -> str:
|
|
37
|
+
if not PROJECT_NAME.match(value):
|
|
38
|
+
raise ValueError(f"{value!r} is not a valid dbt project name")
|
|
39
|
+
return value
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Config(BaseModel):
|
|
43
|
+
model_config = ConfigDict(extra="forbid")
|
|
44
|
+
|
|
45
|
+
store: str | None = None
|
|
46
|
+
projects: list[Upstream] = []
|
|
47
|
+
environments: dict[str, list[str]] = {}
|
|
48
|
+
mesh_dir: str = ".xref"
|
|
49
|
+
|
|
50
|
+
@field_validator("projects", mode="before")
|
|
51
|
+
@classmethod
|
|
52
|
+
def _names_as_strings(cls, value: Any) -> Any:
|
|
53
|
+
items = as_list(value)
|
|
54
|
+
if items is None:
|
|
55
|
+
return value
|
|
56
|
+
return [{"name": v} if isinstance(v, str) else v for v in items]
|
|
57
|
+
|
|
58
|
+
@field_validator("mesh_dir")
|
|
59
|
+
@classmethod
|
|
60
|
+
def _mesh_dir_inside_project(cls, value: str) -> str:
|
|
61
|
+
path = PurePosixPath(value)
|
|
62
|
+
if path.is_absolute() or ".." in path.parts or not path.parts:
|
|
63
|
+
raise ValueError("mesh_dir must be a relative path inside the project, like .xref")
|
|
64
|
+
return value
|
|
65
|
+
|
|
66
|
+
def environments_for(self, upstream: Upstream, environment: str) -> list[str]:
|
|
67
|
+
"""The upstream environments to try, in order, when running in `environment`."""
|
|
68
|
+
if upstream.environment is not None:
|
|
69
|
+
return [upstream.environment]
|
|
70
|
+
if environment in self.environments:
|
|
71
|
+
return self.environments[environment]
|
|
72
|
+
if environment == DEFAULT_ENVIRONMENT:
|
|
73
|
+
return [DEFAULT_ENVIRONMENT]
|
|
74
|
+
return [environment, DEFAULT_ENVIRONMENT]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def load_config(project_dir: Path) -> Config:
|
|
78
|
+
path = project_dir / CONFIG_FILE
|
|
79
|
+
if not path.exists():
|
|
80
|
+
return Config()
|
|
81
|
+
try:
|
|
82
|
+
raw: Any = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
83
|
+
except yaml.YAMLError as exc:
|
|
84
|
+
raise XrefError(f"{path} is not valid YAML: {exc}") from None
|
|
85
|
+
try:
|
|
86
|
+
return Config.model_validate(raw)
|
|
87
|
+
except ValidationError as exc:
|
|
88
|
+
raise XrefError(f"{path} is not a valid xref config:\n{exc}") from None
|
xref/errors.py
ADDED
xref/manifest.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Read the parts of a dbt manifest.json that xref needs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Literal
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
|
11
|
+
|
|
12
|
+
from xref._typing import as_mapping
|
|
13
|
+
from xref.errors import XrefError
|
|
14
|
+
|
|
15
|
+
SUPPORTED_SCHEMA_VERSIONS = frozenset({"https://schemas.getdbt.com/dbt/manifest/v12.json"})
|
|
16
|
+
|
|
17
|
+
PROJECT_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
18
|
+
|
|
19
|
+
Access = Literal["public", "protected", "private"]
|
|
20
|
+
ModelVersion = int | float | str
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Column(BaseModel):
|
|
24
|
+
model_config = ConfigDict(extra="ignore")
|
|
25
|
+
|
|
26
|
+
name: str
|
|
27
|
+
description: str = ""
|
|
28
|
+
data_type: str | None = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ModelNode(BaseModel):
|
|
32
|
+
model_config = ConfigDict(extra="ignore", populate_by_name=True)
|
|
33
|
+
|
|
34
|
+
unique_id: str
|
|
35
|
+
name: str
|
|
36
|
+
package_name: str
|
|
37
|
+
access: Access = "protected"
|
|
38
|
+
database: str | None = None
|
|
39
|
+
schema_name: str = Field(alias="schema")
|
|
40
|
+
alias: str | None = None
|
|
41
|
+
relation_name: str | None = None
|
|
42
|
+
description: str = ""
|
|
43
|
+
columns: dict[str, Column] = Field(default_factory=dict[str, Column])
|
|
44
|
+
version: ModelVersion | None = None
|
|
45
|
+
latest_version: ModelVersion | None = None
|
|
46
|
+
deprecation_date: str | None = None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Metadata(BaseModel):
|
|
50
|
+
model_config = ConfigDict(extra="ignore")
|
|
51
|
+
|
|
52
|
+
dbt_schema_version: str
|
|
53
|
+
dbt_version: str
|
|
54
|
+
generated_at: str
|
|
55
|
+
project_name: str | None = None
|
|
56
|
+
adapter_type: str | None = None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Manifest(BaseModel):
|
|
60
|
+
"""A dbt manifest reduced to the project's own models."""
|
|
61
|
+
|
|
62
|
+
metadata: Metadata
|
|
63
|
+
project_name: str
|
|
64
|
+
models: list[ModelNode]
|
|
65
|
+
|
|
66
|
+
def public_models(self) -> list[ModelNode]:
|
|
67
|
+
return [m for m in self.models if m.access == "public"]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def load_manifest(path: Path) -> Manifest:
|
|
71
|
+
try:
|
|
72
|
+
raw = as_mapping(json.loads(path.read_text(encoding="utf-8")))
|
|
73
|
+
except FileNotFoundError:
|
|
74
|
+
raise XrefError(f"No manifest at {path}. Run `dbt parse` or `dbt build` first.") from None
|
|
75
|
+
except json.JSONDecodeError as exc:
|
|
76
|
+
raise XrefError(f"{path} is not valid JSON: {exc}") from None
|
|
77
|
+
if raw is None:
|
|
78
|
+
raise XrefError(f"{path} is not a dbt manifest.")
|
|
79
|
+
return parse_manifest(raw, source=str(path))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def parse_manifest(raw: dict[str, Any], source: str = "manifest") -> Manifest:
|
|
83
|
+
try:
|
|
84
|
+
metadata = Metadata.model_validate(raw.get("metadata"))
|
|
85
|
+
except ValidationError as exc:
|
|
86
|
+
raise XrefError(f"{source} has no usable metadata block: {exc}") from None
|
|
87
|
+
|
|
88
|
+
if metadata.dbt_schema_version not in SUPPORTED_SCHEMA_VERSIONS:
|
|
89
|
+
supported = ", ".join(sorted(SUPPORTED_SCHEMA_VERSIONS))
|
|
90
|
+
raise XrefError(
|
|
91
|
+
f"{source} uses manifest schema {metadata.dbt_schema_version}; xref supports {supported}."
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
nodes = as_mapping(raw.get("nodes", {}))
|
|
95
|
+
if nodes is None:
|
|
96
|
+
raise XrefError(f"{source} has no nodes.")
|
|
97
|
+
|
|
98
|
+
project_name = metadata.project_name or _infer_project_name(nodes)
|
|
99
|
+
if not PROJECT_NAME.match(project_name):
|
|
100
|
+
raise XrefError(f"{source}: project name {project_name!r} is not a valid dbt project name.")
|
|
101
|
+
|
|
102
|
+
models: list[ModelNode] = []
|
|
103
|
+
for unique_id, value in nodes.items():
|
|
104
|
+
node = as_mapping(value)
|
|
105
|
+
if node is None or node.get("resource_type") != "model":
|
|
106
|
+
continue
|
|
107
|
+
# Models from installed packages (dbt_utils and so on) belong to other projects.
|
|
108
|
+
if node.get("package_name") != project_name:
|
|
109
|
+
continue
|
|
110
|
+
try:
|
|
111
|
+
models.append(ModelNode.model_validate(node))
|
|
112
|
+
except ValidationError as exc:
|
|
113
|
+
raise XrefError(f"{source}: could not read {unique_id}: {exc}") from None
|
|
114
|
+
|
|
115
|
+
models.sort(key=lambda m: m.unique_id)
|
|
116
|
+
return Manifest(metadata=metadata, project_name=project_name, models=models)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _infer_project_name(nodes: dict[str, Any]) -> str:
|
|
120
|
+
"""Older manifests lack metadata.project_name; fall back to the root project's nodes."""
|
|
121
|
+
packages = {
|
|
122
|
+
str(node.get("package_name"))
|
|
123
|
+
for node in map(as_mapping, nodes.values())
|
|
124
|
+
if node is not None and node.get("resource_type") == "model"
|
|
125
|
+
}
|
|
126
|
+
if len(packages) == 1:
|
|
127
|
+
return packages.pop()
|
|
128
|
+
raise XrefError("The manifest has no metadata.project_name and xref could not infer it.")
|
xref/operations.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""The two operations behind the CLI: publish and sync."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import shutil
|
|
6
|
+
import tempfile
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import yaml
|
|
13
|
+
|
|
14
|
+
from xref._typing import as_mapping
|
|
15
|
+
from xref.config import Config
|
|
16
|
+
from xref.errors import XrefError
|
|
17
|
+
from xref.manifest import load_manifest
|
|
18
|
+
from xref.packages import ensure_local_packages, remove_local_packages
|
|
19
|
+
from xref.published import PublishedManifest, from_manifest, parse_published
|
|
20
|
+
from xref.store import ManifestStore, manifest_key
|
|
21
|
+
from xref.stubs import render_stub_package
|
|
22
|
+
|
|
23
|
+
MARKER_FILE = ".xref.json"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class PublishResult:
|
|
28
|
+
published: PublishedManifest
|
|
29
|
+
location: str
|
|
30
|
+
warnings: list[str]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def publish(
|
|
34
|
+
manifest_path: Path,
|
|
35
|
+
store: ManifestStore,
|
|
36
|
+
environment: str,
|
|
37
|
+
now: datetime | None = None,
|
|
38
|
+
) -> PublishResult:
|
|
39
|
+
manifest = load_manifest(manifest_path)
|
|
40
|
+
published, warnings = from_manifest(manifest, environment, now)
|
|
41
|
+
key = manifest_key(published.project_name, environment)
|
|
42
|
+
store.write(key, published.model_dump_json(indent=2).encode())
|
|
43
|
+
return PublishResult(published=published, location=store.describe(key), warnings=warnings)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class SyncedProject:
|
|
48
|
+
published: PublishedManifest
|
|
49
|
+
source: str
|
|
50
|
+
package_dir: Path
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class SyncResult:
|
|
55
|
+
projects: list[SyncedProject]
|
|
56
|
+
packages_file: Path | None
|
|
57
|
+
packages_added: list[str]
|
|
58
|
+
removed: list[str] = field(default_factory=list[str])
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def sync(project_dir: Path, config: Config, store: ManifestStore, environment: str) -> SyncResult:
|
|
62
|
+
if not config.projects:
|
|
63
|
+
raise XrefError(
|
|
64
|
+
f"No upstream projects listed under `projects:` in {project_dir / 'xref.yml'}."
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
own_name = _own_project_name(project_dir)
|
|
68
|
+
mesh_dir = project_dir / config.mesh_dir
|
|
69
|
+
synced: list[SyncedProject] = []
|
|
70
|
+
|
|
71
|
+
# Fetch every upstream before writing anything, so one missing manifest
|
|
72
|
+
# does not leave some packages updated and others stale.
|
|
73
|
+
for upstream in config.projects:
|
|
74
|
+
if upstream.name == own_name:
|
|
75
|
+
raise XrefError(
|
|
76
|
+
f"{upstream.name} is this project's own name; it cannot be its own upstream."
|
|
77
|
+
)
|
|
78
|
+
published, source = _fetch(
|
|
79
|
+
store, upstream.name, config.environments_for(upstream, environment)
|
|
80
|
+
)
|
|
81
|
+
if published.project_name != upstream.name:
|
|
82
|
+
raise XrefError(
|
|
83
|
+
f"{source} holds project {published.project_name!r}, expected {upstream.name!r}."
|
|
84
|
+
)
|
|
85
|
+
synced.append(
|
|
86
|
+
SyncedProject(published=published, source=source, package_dir=mesh_dir / upstream.name)
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
for project in synced:
|
|
90
|
+
_write_package(project.package_dir, project.published, project.source)
|
|
91
|
+
|
|
92
|
+
removed = _remove_stale(project_dir, mesh_dir, {u.name for u in config.projects})
|
|
93
|
+
rel_paths = [p.package_dir.relative_to(project_dir).as_posix() for p in synced]
|
|
94
|
+
packages_path, added = ensure_local_packages(project_dir, rel_paths)
|
|
95
|
+
return SyncResult(
|
|
96
|
+
projects=synced,
|
|
97
|
+
packages_file=packages_path,
|
|
98
|
+
packages_added=added,
|
|
99
|
+
removed=removed,
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _fetch(
|
|
104
|
+
store: ManifestStore, project: str, environments: list[str]
|
|
105
|
+
) -> tuple[PublishedManifest, str]:
|
|
106
|
+
tried: list[str] = []
|
|
107
|
+
for env in environments:
|
|
108
|
+
key = manifest_key(project, env)
|
|
109
|
+
data = store.read(key)
|
|
110
|
+
location = store.describe(key)
|
|
111
|
+
if data is not None:
|
|
112
|
+
return parse_published(data, location), location
|
|
113
|
+
tried.append(location)
|
|
114
|
+
raise XrefError(
|
|
115
|
+
f"No published manifest for {project}. Tried:\n"
|
|
116
|
+
+ "\n".join(f" {t}" for t in tried)
|
|
117
|
+
+ f"\nHas {project} run `xref publish`?"
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _write_package(package_dir: Path, published: PublishedManifest, source: str) -> None:
|
|
122
|
+
if package_dir.exists() and not (package_dir / MARKER_FILE).exists():
|
|
123
|
+
raise XrefError(
|
|
124
|
+
f"{package_dir} exists and was not created by xref; move it before syncing."
|
|
125
|
+
)
|
|
126
|
+
package_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
127
|
+
# Build the new package beside the old one, then swap, so a failed sync
|
|
128
|
+
# never leaves a half-written package behind.
|
|
129
|
+
staging = Path(tempfile.mkdtemp(dir=package_dir.parent, prefix=f".{package_dir.name}-"))
|
|
130
|
+
try:
|
|
131
|
+
for rel, content in render_stub_package(published).items():
|
|
132
|
+
target = staging / rel
|
|
133
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
134
|
+
target.write_text(content, encoding="utf-8")
|
|
135
|
+
marker: dict[str, Any] = {
|
|
136
|
+
"source": source,
|
|
137
|
+
"environment": published.environment,
|
|
138
|
+
"generated_at": published.generated_at,
|
|
139
|
+
"published_at": published.published_at,
|
|
140
|
+
}
|
|
141
|
+
(staging / MARKER_FILE).write_text(
|
|
142
|
+
yaml.safe_dump(marker, sort_keys=False), encoding="utf-8"
|
|
143
|
+
)
|
|
144
|
+
if package_dir.exists():
|
|
145
|
+
shutil.rmtree(package_dir)
|
|
146
|
+
staging.rename(package_dir)
|
|
147
|
+
except BaseException:
|
|
148
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
149
|
+
raise
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _remove_stale(project_dir: Path, mesh_dir: Path, keep: set[str]) -> list[str]:
|
|
153
|
+
"""Delete stub packages for projects no longer in xref.yml, and their packages entries."""
|
|
154
|
+
if not mesh_dir.exists():
|
|
155
|
+
return []
|
|
156
|
+
stale = [
|
|
157
|
+
child
|
|
158
|
+
for child in sorted(mesh_dir.iterdir())
|
|
159
|
+
if child.is_dir() and child.name not in keep and (child / MARKER_FILE).exists()
|
|
160
|
+
]
|
|
161
|
+
# Remove the packages entries first: if that fails, the folders stay and dbt deps still works.
|
|
162
|
+
remove_local_packages(project_dir, [c.relative_to(project_dir).as_posix() for c in stale])
|
|
163
|
+
for child in stale:
|
|
164
|
+
shutil.rmtree(child)
|
|
165
|
+
return [c.name for c in stale]
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _own_project_name(project_dir: Path) -> str | None:
|
|
169
|
+
path = project_dir / "dbt_project.yml"
|
|
170
|
+
if not path.exists():
|
|
171
|
+
raise XrefError(
|
|
172
|
+
f"No dbt_project.yml in {project_dir}. Run xref from the dbt project directory or pass --project-dir."
|
|
173
|
+
)
|
|
174
|
+
try:
|
|
175
|
+
data = as_mapping(yaml.safe_load(path.read_text(encoding="utf-8")) or {})
|
|
176
|
+
except yaml.YAMLError as exc:
|
|
177
|
+
raise XrefError(f"{path} is not valid YAML: {exc}") from None
|
|
178
|
+
name = data.get("name") if data is not None else None
|
|
179
|
+
return name if isinstance(name, str) else None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def age_of(timestamp: str, now: datetime | None = None) -> str:
|
|
183
|
+
"""'3h ago' style age for a manifest timestamp, or the raw value if unparseable."""
|
|
184
|
+
try:
|
|
185
|
+
then = datetime.fromisoformat(timestamp)
|
|
186
|
+
except ValueError:
|
|
187
|
+
return timestamp
|
|
188
|
+
if then.tzinfo is None:
|
|
189
|
+
then = then.replace(tzinfo=UTC)
|
|
190
|
+
seconds = int(((now or datetime.now(UTC)) - then).total_seconds())
|
|
191
|
+
if seconds < 0:
|
|
192
|
+
return "just now"
|
|
193
|
+
for unit, size in (("d", 86400), ("h", 3600), ("m", 60)):
|
|
194
|
+
if seconds >= size:
|
|
195
|
+
return f"{seconds // size}{unit} ago"
|
|
196
|
+
return f"{seconds}s ago"
|