interloper-toolkit 0.100.0__tar.gz → 0.102.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/PKG-INFO +2 -2
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/README.md +1 -1
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/pyproject.toml +12 -1
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/pyproject.toml.orig +10 -3
- interloper_toolkit-0.102.0/src/interloper_toolkit/analytics.py +119 -0
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/authz.py +9 -11
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/catalog.py +5 -1
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/collection.py +12 -6
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/lineage.py +38 -27
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/models.py +147 -103
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/scheduling.py +93 -154
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/stats.py +2 -20
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/tools.py +2 -4
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/utils.py +1 -0
- interloper_toolkit-0.100.0/src/interloper_toolkit/analytics.py +0 -360
- interloper_toolkit-0.100.0/src/interloper_toolkit/errors.py +0 -110
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/__init__.py +0 -0
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/context.py +0 -0
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/jobs.py +0 -0
- {interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/sources.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: interloper-toolkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.102.0
|
|
4
4
|
Summary: Interloper tool functions shared by the AI surfaces (agent, MCP)
|
|
5
5
|
Author: Guillaume Onfroy
|
|
6
6
|
Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
|
|
@@ -42,7 +42,7 @@ Modules:
|
|
|
42
42
|
- `sources` and `jobs`: creating sources, resolving their fields, creating jobs
|
|
43
43
|
- `lineage`: dependency analysis, impact assessment, DAG traversal
|
|
44
44
|
- `scheduling`: jobs, runs and backfills, monitoring and control
|
|
45
|
-
- `analytics`: run statistics, partition coverage
|
|
45
|
+
- `analytics`: job health, run statistics, partition coverage
|
|
46
46
|
- `tools`: the table above
|
|
47
47
|
|
|
48
48
|
Depends only on `interloper-core` and `interloper-db`; no LLM-framework
|
|
@@ -31,7 +31,7 @@ Modules:
|
|
|
31
31
|
- `sources` and `jobs`: creating sources, resolving their fields, creating jobs
|
|
32
32
|
- `lineage`: dependency analysis, impact assessment, DAG traversal
|
|
33
33
|
- `scheduling`: jobs, runs and backfills, monitoring and control
|
|
34
|
-
- `analytics`: run statistics, partition coverage
|
|
34
|
+
- `analytics`: job health, run statistics, partition coverage
|
|
35
35
|
- `tools`: the table above
|
|
36
36
|
|
|
37
37
|
Depends only on `interloper-core` and `interloper-db`; no LLM-framework
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "interloper-toolkit"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.102.0"
|
|
4
4
|
description = "Interloper tool functions shared by the AI surfaces (agent, MCP)"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
@@ -37,17 +37,28 @@ extend-select = [
|
|
|
37
37
|
"ANN202",
|
|
38
38
|
"DOC",
|
|
39
39
|
"D",
|
|
40
|
+
"TID251",
|
|
40
41
|
]
|
|
41
42
|
|
|
42
43
|
[tool.ruff.lint.pydocstyle]
|
|
43
44
|
convention = "google"
|
|
44
45
|
|
|
46
|
+
[tool.ruff.lint.flake8-tidy-imports.banned-api.sqlalchemy]
|
|
47
|
+
msg = "Database access goes through interloper_db.Store facets"
|
|
48
|
+
|
|
49
|
+
[tool.ruff.lint.flake8-tidy-imports.banned-api.sqlmodel]
|
|
50
|
+
msg = "Database access goes through interloper_db.Store facets"
|
|
51
|
+
|
|
52
|
+
[tool.ruff.lint.flake8-tidy-imports.banned-api."interloper_db.session"]
|
|
53
|
+
msg = "Database access goes through interloper_db.Store facets"
|
|
54
|
+
|
|
45
55
|
[tool.ruff.lint.per-file-ignores]
|
|
46
56
|
"__init__.py" = [
|
|
47
57
|
"F401",
|
|
48
58
|
"F403",
|
|
49
59
|
]
|
|
50
60
|
"tests/**" = [
|
|
61
|
+
"TID251",
|
|
51
62
|
"ANN",
|
|
52
63
|
"F811",
|
|
53
64
|
"D101",
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# ###############
|
|
4
4
|
[project]
|
|
5
5
|
name = "interloper-toolkit"
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "0.102.0"
|
|
7
7
|
description = "Interloper tool functions shared by the AI surfaces (agent, MCP)"
|
|
8
8
|
readme = "README.md"
|
|
9
9
|
authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]
|
|
@@ -29,12 +29,19 @@ line-length = 120
|
|
|
29
29
|
|
|
30
30
|
[tool.ruff.lint]
|
|
31
31
|
preview = true
|
|
32
|
-
extend-select = ["E", "I", "UP", "ANN001", "ANN201", "ANN202", "DOC", "D"]
|
|
32
|
+
extend-select = ["E", "I", "UP", "ANN001", "ANN201", "ANN202", "DOC", "D", "TID251"]
|
|
33
33
|
|
|
34
34
|
[tool.ruff.lint.pydocstyle]
|
|
35
35
|
convention = "google"
|
|
36
36
|
|
|
37
|
+
# Raw database access lives in interloper-db alone; this package reads and
|
|
38
|
+
# writes through the store's facets.
|
|
39
|
+
[tool.ruff.lint.flake8-tidy-imports.banned-api]
|
|
40
|
+
"sqlalchemy".msg = "Database access goes through interloper_db.Store facets"
|
|
41
|
+
"sqlmodel".msg = "Database access goes through interloper_db.Store facets"
|
|
42
|
+
"interloper_db.session".msg = "Database access goes through interloper_db.Store facets"
|
|
43
|
+
|
|
37
44
|
[tool.ruff.lint.per-file-ignores]
|
|
38
45
|
"__init__.py" = ["F401", "F403"]
|
|
39
|
-
"tests/**" = ["ANN", "F811", "D101", "D102", "D103", "D104", "RUF069", "PLW0108"]
|
|
46
|
+
"tests/**" = ["TID251", "ANN", "F811", "D101", "D102", "D103", "D104", "RUF069", "PLW0108"]
|
|
40
47
|
"src/interloper_toolkit/{analytics,catalog,collection,jobs,lineage,scheduling,sources}.py" = ["D417", "DOC201", "BLE001"]
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Analytics tools: job health, run statistics and partition coverage, read off ``store.insights``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime
|
|
6
|
+
from uuid import UUID
|
|
7
|
+
|
|
8
|
+
from interloper_toolkit.context import ToolkitContext
|
|
9
|
+
from interloper_toolkit.models import (
|
|
10
|
+
AssetCoverage,
|
|
11
|
+
JobHealthReport,
|
|
12
|
+
JobHealthRow,
|
|
13
|
+
RunStats,
|
|
14
|
+
ToolError,
|
|
15
|
+
)
|
|
16
|
+
from interloper_toolkit.stats import window
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def job_health(ctx: ToolkitContext, limit: int = 50, offset: int = 0) -> JobHealthReport | ToolError:
|
|
20
|
+
"""Every job's health: failing, overdue, last success and next firing.
|
|
21
|
+
|
|
22
|
+
A job is failing when it is enabled and its latest attempt failed, and
|
|
23
|
+
overdue when its scheduled slot passed more than 15 minutes ago without
|
|
24
|
+
the scheduler firing it.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
limit: Maximum number of jobs to return (default 50).
|
|
28
|
+
offset: Number of jobs to skip, for paging past the first page.
|
|
29
|
+
|
|
30
|
+
Returns the page of jobs, failing then overdue ones first, each with its
|
|
31
|
+
latest run, last success, next firing and the partitions that firing
|
|
32
|
+
covers, plus the org-wide failing and overdue counts.
|
|
33
|
+
"""
|
|
34
|
+
try:
|
|
35
|
+
health = ctx.store.insights.health(ctx.org_id, now=datetime.datetime.now(tz=datetime.timezone.utc))
|
|
36
|
+
rows = sorted(
|
|
37
|
+
(JobHealthRow.from_health(job) for job in health.jobs),
|
|
38
|
+
key=lambda row: (not row.failing, not row.overdue, row.job_name or ""),
|
|
39
|
+
)
|
|
40
|
+
page = rows[offset : offset + limit]
|
|
41
|
+
return JobHealthReport(
|
|
42
|
+
failing=sum(row.failing for row in rows),
|
|
43
|
+
overdue=sum(row.overdue for row in rows),
|
|
44
|
+
count=len(page),
|
|
45
|
+
total=len(rows),
|
|
46
|
+
jobs=page,
|
|
47
|
+
)
|
|
48
|
+
except Exception as e:
|
|
49
|
+
return ToolError(error=str(e))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def run_stats(
|
|
53
|
+
ctx: ToolkitContext,
|
|
54
|
+
since: str | None = None,
|
|
55
|
+
until: str | None = None,
|
|
56
|
+
component_id: str | None = None,
|
|
57
|
+
limit: int = 25,
|
|
58
|
+
offset: int = 0,
|
|
59
|
+
) -> RunStats | ToolError:
|
|
60
|
+
"""Per-job run statistics over a window: verdicts, durations and retries.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
since: ISO date or datetime the window opens at (default: 7 days ago).
|
|
64
|
+
until: ISO date or datetime the window closes before (default: open).
|
|
65
|
+
component_id: Restrict to this job UUID.
|
|
66
|
+
limit: Maximum number of jobs to return (default 25).
|
|
67
|
+
offset: Number of jobs to skip, for paging past the first page.
|
|
68
|
+
|
|
69
|
+
Returns one row per job that ran in the window, most failures first: run
|
|
70
|
+
stacks by final status, attempts, duration p50/p90/max in seconds, how
|
|
71
|
+
many stacks were retried and of those how many healed or are still
|
|
72
|
+
failing, and the last success.
|
|
73
|
+
"""
|
|
74
|
+
try:
|
|
75
|
+
start, end = window(since, until, default_days=7)
|
|
76
|
+
job_id = UUID(component_id) if component_id else None
|
|
77
|
+
outcomes = ctx.store.insights.outcomes(ctx.org_id, since=start, until=end, job_id=job_id)
|
|
78
|
+
page = outcomes[offset : offset + limit]
|
|
79
|
+
return RunStats(since=start, until=end, count=len(page), total=len(outcomes), jobs=page)
|
|
80
|
+
except Exception as e:
|
|
81
|
+
return ToolError(error=str(e))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def asset_coverage(
|
|
85
|
+
ctx: ToolkitContext,
|
|
86
|
+
component_id: str,
|
|
87
|
+
start_key: str,
|
|
88
|
+
end_key: str,
|
|
89
|
+
limit: int = 50,
|
|
90
|
+
offset: int = 0,
|
|
91
|
+
) -> AssetCoverage | ToolError:
|
|
92
|
+
"""Per-asset partition coverage of a job over a range of partition keys.
|
|
93
|
+
|
|
94
|
+
An asset counts as covered for a partition once any run's execution of it
|
|
95
|
+
succeeded, whatever the run targeted, so a run where most assets
|
|
96
|
+
succeeded shows what is actually missing. Every asset the job targets
|
|
97
|
+
appears, directly or through a source it targets.
|
|
98
|
+
|
|
99
|
+
Args:
|
|
100
|
+
component_id: UUID of the job.
|
|
101
|
+
start_key: First partition key, in the job's granularity (2026-07-01,
|
|
102
|
+
2026-07, 2026, or 2026-07-01T13).
|
|
103
|
+
end_key: Last partition key, inclusive; must share the start key's
|
|
104
|
+
granularity.
|
|
105
|
+
limit: Maximum number of assets to return (default 50).
|
|
106
|
+
offset: Number of assets to skip, for paging past the first page.
|
|
107
|
+
|
|
108
|
+
Returns the page of assets, least covered first, each with its covered,
|
|
109
|
+
failed and never-run partition counts and its missing partitions as
|
|
110
|
+
ranges ready for a backfill, plus a rollup of the range's partitions by
|
|
111
|
+
whether all, some or none of the assets are covered.
|
|
112
|
+
"""
|
|
113
|
+
try:
|
|
114
|
+
coverage = ctx.store.insights.coverage_by_key(
|
|
115
|
+
ctx.org_id, UUID(component_id), start_key=start_key, end_key=end_key
|
|
116
|
+
)
|
|
117
|
+
return AssetCoverage.from_coverage(coverage, limit=limit, offset=offset)
|
|
118
|
+
except Exception as e:
|
|
119
|
+
return ToolError(error=str(e))
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
The context carries the caller's role in the organisation; a write declares
|
|
4
4
|
the role it needs and refuses, as a structured :class:`ToolError`, before its
|
|
5
|
-
body runs. The ranks
|
|
6
|
-
a caller can do through a tool exactly what it could do
|
|
5
|
+
body runs. The ranks are :class:`~interloper_db.Role`'s, the ones the API's
|
|
6
|
+
route gates apply, so a caller can do through a tool exactly what it could do
|
|
7
|
+
through the app.
|
|
7
8
|
"""
|
|
8
9
|
|
|
9
10
|
from __future__ import annotations
|
|
@@ -13,11 +14,11 @@ import inspect
|
|
|
13
14
|
from collections.abc import Callable
|
|
14
15
|
from typing import Any, TypeVar, cast
|
|
15
16
|
|
|
17
|
+
from interloper_db.models import Role
|
|
18
|
+
|
|
16
19
|
from interloper_toolkit.context import ToolkitContext
|
|
17
20
|
from interloper_toolkit.models import ToolError
|
|
18
21
|
|
|
19
|
-
_ROLE_RANK = {"viewer": 0, "editor": 1, "admin": 2}
|
|
20
|
-
|
|
21
22
|
F = TypeVar("F", bound=Callable[..., Any])
|
|
22
23
|
|
|
23
24
|
|
|
@@ -29,16 +30,13 @@ def requires_role(minimum: str) -> Callable[[F], F]:
|
|
|
29
30
|
function keeps its name and docstring, which the AI surfaces adopt.
|
|
30
31
|
|
|
31
32
|
Args:
|
|
32
|
-
minimum: The lowest role allowed: ``viewer``, ``editor`` or ``admin
|
|
33
|
+
minimum: The lowest role allowed: ``viewer``, ``editor`` or ``admin``;
|
|
34
|
+
any other name is a ``ConfigError`` at decoration time.
|
|
33
35
|
|
|
34
36
|
Returns:
|
|
35
37
|
The decorator to apply to a tool function.
|
|
36
|
-
|
|
37
|
-
Raises:
|
|
38
|
-
ValueError: If *minimum* is not a known role.
|
|
39
38
|
"""
|
|
40
|
-
|
|
41
|
-
raise ValueError(f"Unknown role {minimum!r}; expected one of {sorted(_ROLE_RANK)}")
|
|
39
|
+
Role.parse(minimum)
|
|
42
40
|
|
|
43
41
|
def decorate(func: F) -> F:
|
|
44
42
|
if inspect.iscoroutinefunction(func):
|
|
@@ -68,6 +66,6 @@ def denied(role: str, minimum: str) -> ToolError | None:
|
|
|
68
66
|
Returns:
|
|
69
67
|
The structured error, or ``None`` when the call may proceed.
|
|
70
68
|
"""
|
|
71
|
-
if
|
|
69
|
+
if not Role.at_least(role, minimum):
|
|
72
70
|
return ToolError(error=f"Requires {minimum} role or higher")
|
|
73
71
|
return None
|
|
@@ -10,6 +10,7 @@ from __future__ import annotations
|
|
|
10
10
|
from typing import Any
|
|
11
11
|
|
|
12
12
|
from interloper.oauth import OAuthAppCredentials
|
|
13
|
+
from interloper_db import ComponentQuery
|
|
13
14
|
|
|
14
15
|
from interloper_toolkit.context import ToolkitContext
|
|
15
16
|
from interloper_toolkit.models import (
|
|
@@ -86,7 +87,10 @@ def list_definitions(
|
|
|
86
87
|
|
|
87
88
|
collection_counts: dict[str, int] = {}
|
|
88
89
|
if kind == "source":
|
|
89
|
-
|
|
90
|
+
sources = ctx.store.components.list(
|
|
91
|
+
ctx.org_id, ComponentQuery(kind=["source"], roots_only=False, limit=None)
|
|
92
|
+
)
|
|
93
|
+
for s in sources.items:
|
|
90
94
|
collection_counts[s.key] = collection_counts.get(s.key, 0) + 1
|
|
91
95
|
|
|
92
96
|
results = []
|
{interloper_toolkit-0.100.0 → interloper_toolkit-0.102.0}/src/interloper_toolkit/collection.py
RENAMED
|
@@ -31,6 +31,7 @@ from interloper.errors import (
|
|
|
31
31
|
from interloper.oauth import OAuthAppCredentials
|
|
32
32
|
from interloper.settings import AppSettings
|
|
33
33
|
from interloper.utils.concurrency import invoke
|
|
34
|
+
from interloper_db import ComponentQuery
|
|
34
35
|
from pydantic import ValidationError
|
|
35
36
|
|
|
36
37
|
from interloper_toolkit.authz import requires_role
|
|
@@ -81,7 +82,8 @@ def list_components(
|
|
|
81
82
|
try:
|
|
82
83
|
if kind is None:
|
|
83
84
|
counts: dict[str, int] = {}
|
|
84
|
-
|
|
85
|
+
everything = ctx.store.components.list(ctx.org_id, ComponentQuery(q=q, roots_only=False, limit=None))
|
|
86
|
+
for c in everything.items:
|
|
85
87
|
counts[c.kind] = counts.get(c.kind, 0) + 1
|
|
86
88
|
return ComponentCounts(
|
|
87
89
|
component_counts=counts,
|
|
@@ -91,8 +93,10 @@ def list_components(
|
|
|
91
93
|
return ToolError(error=f"Unknown kind '{kind}'", valid_values=sorted(KINDS.keys()))
|
|
92
94
|
|
|
93
95
|
results = []
|
|
94
|
-
|
|
95
|
-
|
|
96
|
+
components = ctx.store.components.list(
|
|
97
|
+
ctx.org_id, ComponentQuery(kind=[kind], q=q, roots_only=False, limit=limit, offset=offset)
|
|
98
|
+
)
|
|
99
|
+
for c in components.items:
|
|
96
100
|
entry = ComponentSummary(
|
|
97
101
|
id=str(c.id),
|
|
98
102
|
key=c.key,
|
|
@@ -107,7 +111,7 @@ def list_components(
|
|
|
107
111
|
entry.asset_count = len(c.children)
|
|
108
112
|
results.append(entry)
|
|
109
113
|
|
|
110
|
-
return ComponentList(kind=kind, count=len(results), total=total, components=results)
|
|
114
|
+
return ComponentList(kind=kind, count=len(results), total=components.total, components=results)
|
|
111
115
|
except Exception as e:
|
|
112
116
|
return ToolError(error=str(e))
|
|
113
117
|
|
|
@@ -238,7 +242,7 @@ def unbind_relation(ctx: ToolkitContext, component_id: str, name: str, dst_id: s
|
|
|
238
242
|
"""
|
|
239
243
|
try:
|
|
240
244
|
src = ctx.store.components.get(UUID(component_id), org_id=ctx.org_id)
|
|
241
|
-
ctx.store.relations.
|
|
245
|
+
ctx.store.relations.delete(src.id, name=name, dst_id=UUID(dst_id))
|
|
242
246
|
except (ConfigError, NotFoundError, ValueError) as e:
|
|
243
247
|
return ToolError(error=str(e))
|
|
244
248
|
return UnbindResult(src_id=component_id, name=name, dst_id=dst_id)
|
|
@@ -298,7 +302,9 @@ def request_connection_setup(
|
|
|
298
302
|
if not force_new:
|
|
299
303
|
existing = [
|
|
300
304
|
ComponentRef(id=c.id, kind=c.kind, key=c.key, name=c.name)
|
|
301
|
-
for c in ctx.store.components.
|
|
305
|
+
for c in ctx.store.components.list(
|
|
306
|
+
ctx.org_id, ComponentQuery(kind=["connection"], roots_only=False, limit=None)
|
|
307
|
+
).items
|
|
302
308
|
if c.key == connection_key
|
|
303
309
|
]
|
|
304
310
|
if existing:
|
|
@@ -5,6 +5,8 @@ from __future__ import annotations
|
|
|
5
5
|
from collections import defaultdict
|
|
6
6
|
from uuid import UUID
|
|
7
7
|
|
|
8
|
+
from interloper_db import ComponentQuery, RelationQuery
|
|
9
|
+
|
|
8
10
|
from interloper_toolkit.context import ToolkitContext
|
|
9
11
|
from interloper_toolkit.models import (
|
|
10
12
|
AssetRef,
|
|
@@ -19,6 +21,9 @@ from interloper_toolkit.models import (
|
|
|
19
21
|
UpstreamResult,
|
|
20
22
|
)
|
|
21
23
|
|
|
24
|
+
_ASSET_EDGES = RelationQuery(src_kind="asset", dst_kind="asset", limit=None)
|
|
25
|
+
_ASSETS = ComponentQuery(kind=["asset"], roots_only=False, limit=None)
|
|
26
|
+
|
|
22
27
|
|
|
23
28
|
def get_upstream(ctx: ToolkitContext, asset_id: str) -> UpstreamResult | ToolError:
|
|
24
29
|
"""Get the direct upstream dependencies of an asset.
|
|
@@ -31,18 +36,20 @@ def get_upstream(ctx: ToolkitContext, asset_id: str) -> UpstreamResult | ToolErr
|
|
|
31
36
|
"""
|
|
32
37
|
try:
|
|
33
38
|
target = ctx.store.components.get(UUID(asset_id), kind="asset", org_id=ctx.org_id).id
|
|
34
|
-
deps = ctx.store.relations.
|
|
39
|
+
deps = ctx.store.relations.list(ctx.org_id, _ASSET_EDGES).items
|
|
35
40
|
|
|
36
41
|
upstream = []
|
|
37
42
|
for dep in deps:
|
|
38
43
|
if dep.src_id == target:
|
|
39
|
-
asset = ctx.store.components.get(dep.dst_id, kind="asset")
|
|
40
|
-
upstream.append(
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
44
|
+
asset = ctx.store.components.get(dep.dst_id, org_id=ctx.org_id, kind="asset")
|
|
45
|
+
upstream.append(
|
|
46
|
+
RelationEdge(
|
|
47
|
+
asset_id=str(dep.dst_id),
|
|
48
|
+
param_name=dep.name,
|
|
49
|
+
asset_key=asset.key,
|
|
50
|
+
source_id=str(asset.parent_id),
|
|
51
|
+
)
|
|
52
|
+
)
|
|
46
53
|
|
|
47
54
|
return UpstreamResult(asset_id=asset_id, upstream=upstream)
|
|
48
55
|
except Exception as e:
|
|
@@ -59,18 +66,20 @@ def get_downstream(ctx: ToolkitContext, asset_id: str) -> DownstreamResult | Too
|
|
|
59
66
|
"""
|
|
60
67
|
try:
|
|
61
68
|
target = ctx.store.components.get(UUID(asset_id), kind="asset", org_id=ctx.org_id).id
|
|
62
|
-
deps = ctx.store.relations.
|
|
69
|
+
deps = ctx.store.relations.list(ctx.org_id, _ASSET_EDGES).items
|
|
63
70
|
|
|
64
71
|
downstream = []
|
|
65
72
|
for dep in deps:
|
|
66
73
|
if dep.dst_id == target:
|
|
67
|
-
asset = ctx.store.components.get(dep.src_id, kind="asset")
|
|
68
|
-
downstream.append(
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
+
asset = ctx.store.components.get(dep.src_id, org_id=ctx.org_id, kind="asset")
|
|
75
|
+
downstream.append(
|
|
76
|
+
RelationEdge(
|
|
77
|
+
asset_id=str(dep.src_id),
|
|
78
|
+
param_name=dep.name,
|
|
79
|
+
asset_key=asset.key,
|
|
80
|
+
source_id=str(asset.parent_id),
|
|
81
|
+
)
|
|
82
|
+
)
|
|
74
83
|
|
|
75
84
|
return DownstreamResult(asset_id=asset_id, downstream=downstream)
|
|
76
85
|
except Exception as e:
|
|
@@ -165,8 +174,8 @@ def cross_source_dependencies(ctx: ToolkitContext) -> CrossSourceDependencies |
|
|
|
165
174
|
different sources.
|
|
166
175
|
"""
|
|
167
176
|
try:
|
|
168
|
-
deps = ctx.store.relations.
|
|
169
|
-
assets = ctx.store.components.
|
|
177
|
+
deps = ctx.store.relations.list(ctx.org_id, _ASSET_EDGES).items
|
|
178
|
+
assets = ctx.store.components.list(ctx.org_id, _ASSETS).items
|
|
170
179
|
|
|
171
180
|
asset_source: dict[UUID, UUID | None] = {}
|
|
172
181
|
asset_info: dict[UUID, AssetRef] = {}
|
|
@@ -181,13 +190,15 @@ def cross_source_dependencies(ctx: ToolkitContext) -> CrossSourceDependencies |
|
|
|
181
190
|
src_down = asset_source.get(dep.src_id)
|
|
182
191
|
src_up = asset_source.get(dep.dst_id)
|
|
183
192
|
if src_down and src_up and src_down != src_up:
|
|
184
|
-
cross_deps.append(
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
193
|
+
cross_deps.append(
|
|
194
|
+
CrossSourceEdge(
|
|
195
|
+
downstream_asset_id=str(dep.src_id),
|
|
196
|
+
downstream=asset_info.get(dep.src_id, AssetRef()),
|
|
197
|
+
upstream_asset_id=str(dep.dst_id),
|
|
198
|
+
upstream=asset_info.get(dep.dst_id, AssetRef()),
|
|
199
|
+
param_name=dep.name,
|
|
200
|
+
)
|
|
201
|
+
)
|
|
191
202
|
|
|
192
203
|
return CrossSourceDependencies(cross_source_count=len(cross_deps), dependencies=cross_deps)
|
|
193
204
|
except Exception as e:
|
|
@@ -211,8 +222,8 @@ def _build_adjacency(
|
|
|
211
222
|
``(adjacency_map, asset_info_map)`` — info values feed
|
|
212
223
|
:class:`LineageItem` kwargs (asset_key, source_id, source_key).
|
|
213
224
|
"""
|
|
214
|
-
deps = ctx.store.relations.
|
|
215
|
-
assets = ctx.store.components.
|
|
225
|
+
deps = ctx.store.relations.list(ctx.org_id, _ASSET_EDGES).items
|
|
226
|
+
assets = ctx.store.components.list(ctx.org_id, _ASSETS).items
|
|
216
227
|
|
|
217
228
|
asset_info: dict[UUID, dict[str, str]] = {}
|
|
218
229
|
source_keys: dict[UUID, str] = {}
|