contextbase-core 0.0.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contextbase_core-0.0.0a1.dist-info/METADATA +33 -0
- contextbase_core-0.0.0a1.dist-info/RECORD +9 -0
- contextbase_core-0.0.0a1.dist-info/WHEEL +4 -0
- core/__init__.py +1 -0
- core/definitions.py +147 -0
- core/defs/__init__.py +0 -0
- core/defs/binding_partitions.py +72 -0
- core/defs/binding_sync_state.py +221 -0
- pyproject.toml +84 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: contextbase-core
|
|
3
|
+
Version: 0.0.0a1
|
|
4
|
+
Requires-Python: <3.13,>=3.12
|
|
5
|
+
Requires-Dist: contextbase-plugin-asana==0.0.0-alpha-1
|
|
6
|
+
Requires-Dist: contextbase-plugin-chrome-local==0.0.0-alpha-1
|
|
7
|
+
Requires-Dist: contextbase-plugin-claude-local==0.0.0-alpha-1
|
|
8
|
+
Requires-Dist: contextbase-plugin-clickup==0.0.0-alpha-1
|
|
9
|
+
Requires-Dist: contextbase-plugin-codex-local==0.0.0-alpha-1
|
|
10
|
+
Requires-Dist: contextbase-plugin-github==0.0.0-alpha-1
|
|
11
|
+
Requires-Dist: contextbase-plugin-gmail==0.0.0-alpha-1
|
|
12
|
+
Requires-Dist: contextbase-plugin-gong==0.0.0-alpha-1
|
|
13
|
+
Requires-Dist: contextbase-plugin-google-calendar==0.0.0-alpha-1
|
|
14
|
+
Requires-Dist: contextbase-plugin-granola==0.0.0-alpha-1
|
|
15
|
+
Requires-Dist: contextbase-plugin-hubspot==0.0.0-alpha-1
|
|
16
|
+
Requires-Dist: contextbase-plugin-icloud-tabs-local==0.0.0-alpha-1
|
|
17
|
+
Requires-Dist: contextbase-plugin-imessage-local==0.0.0-alpha-1
|
|
18
|
+
Requires-Dist: contextbase-plugin-intercom==0.0.0-alpha-1
|
|
19
|
+
Requires-Dist: contextbase-plugin-jira==0.0.0-alpha-1
|
|
20
|
+
Requires-Dist: contextbase-plugin-linear==0.0.0-alpha-1
|
|
21
|
+
Requires-Dist: contextbase-plugin-microsoft-calendar==0.0.0-alpha-1
|
|
22
|
+
Requires-Dist: contextbase-plugin-microsoft-mail==0.0.0-alpha-1
|
|
23
|
+
Requires-Dist: contextbase-plugin-microsoft-planner-premium==0.0.0-alpha-1
|
|
24
|
+
Requires-Dist: contextbase-plugin-notion==0.0.0-alpha-1
|
|
25
|
+
Requires-Dist: contextbase-plugin-reminders-local==0.0.0-alpha-1
|
|
26
|
+
Requires-Dist: contextbase-plugin-salesforce==0.0.0-alpha-1
|
|
27
|
+
Requires-Dist: contextbase-plugin-slack==0.0.0-alpha-1
|
|
28
|
+
Requires-Dist: contextbase-plugin-whatsapp-local==0.0.0-alpha-1
|
|
29
|
+
Requires-Dist: contextbase-plugin-workflowy==0.0.0-alpha-1
|
|
30
|
+
Requires-Dist: contextbase-shared-plugins==0.0.0-alpha-1
|
|
31
|
+
Requires-Dist: dagster==1.12.14
|
|
32
|
+
Requires-Dist: httpx>=0.28.1
|
|
33
|
+
Requires-Dist: pydantic-settings>=2.11.0
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
core/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
2
|
+
core/definitions.py,sha256=FvQnW_luoVSkt9ypvFYw47HsSuSVzqvnWd_F-kmUXhw,5950
|
|
3
|
+
core/defs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
core/defs/binding_partitions.py,sha256=HTU9Nzp84M1qF7h2B8kzgKHqoldorBAqCZTzggWDVLk,2767
|
|
5
|
+
core/defs/binding_sync_state.py,sha256=vUY8OsU6Q6pQFMhpuCn-y7eDNALbwkX9WLvp80IuClE,8196
|
|
6
|
+
pyproject.toml,sha256=rKFyq3-MxUhxDdHe-k9VefWjIj0i1EDkWfXgZuMmIS0,3313
|
|
7
|
+
contextbase_core-0.0.0a1.dist-info/METADATA,sha256=EGmDNHdx2quXObvskB5dP5xVJJhRdgzqZEnVloyDTQ4,1757
|
|
8
|
+
contextbase_core-0.0.0a1.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
9
|
+
contextbase_core-0.0.0a1.dist-info/RECORD,,
|
core/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
core/definitions.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
from importlib import import_module
|
|
2
|
+
from importlib.metadata import entry_points
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from dagster import (
|
|
6
|
+
Definitions,
|
|
7
|
+
ResourceDefinition,
|
|
8
|
+
build_defs_for_component,
|
|
9
|
+
definitions,
|
|
10
|
+
load_from_defs_folder,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
from shared_plugins.base_platform import BasePlatformClient
|
|
14
|
+
from shared_plugins.env import (
|
|
15
|
+
CTXB_BASE_ID_ENV_VAR,
|
|
16
|
+
CTX_PLATFORM_URL_ENV_VAR,
|
|
17
|
+
load_shared_python_settings,
|
|
18
|
+
)
|
|
19
|
+
from shared_plugins.machine_token import resolve_machine_token_auth
|
|
20
|
+
from shared_plugins.provider_token import ProviderTokenClient, ProviderTokenSettings
|
|
21
|
+
from shared_plugins.sentry import init_sentry
|
|
22
|
+
|
|
23
|
+
# Initialize Sentry as early as possible. This module is the single entry every
|
|
24
|
+
# Dagster process loads (daemon, per-run subprocess, per-step subprocess; see
|
|
25
|
+
# base-cli dagster/command.ts DAGSTER_CORE_MODULE_NAME), so initializing here
|
|
26
|
+
# self-arms error reporting in all of them. Idempotent within a process.
|
|
27
|
+
init_sentry()
|
|
28
|
+
|
|
29
|
+
# Plugins advertise their Component class via this entry-point group (one per
|
|
30
|
+
# installed contextbase plugin). Discovery is dist-metadata based, so it works
|
|
31
|
+
# identically for editable workspace installs and wheels in site-packages.
|
|
32
|
+
PLUGIN_ENTRY_POINT_GROUP = "contextlayer.plugins"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _without_component_tree(definitions: Definitions) -> Definitions:
|
|
36
|
+
return Definitions(
|
|
37
|
+
assets=definitions.assets,
|
|
38
|
+
schedules=definitions.schedules,
|
|
39
|
+
sensors=definitions.sensors,
|
|
40
|
+
jobs=definitions.jobs,
|
|
41
|
+
resources=definitions.resources,
|
|
42
|
+
executor=definitions.executor,
|
|
43
|
+
loggers=definitions.loggers,
|
|
44
|
+
asset_checks=definitions.asset_checks,
|
|
45
|
+
metadata=definitions.metadata,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _load_core_defs() -> Definitions:
|
|
50
|
+
# core stays the single Dagster project. Its own defs (the binding-partitions
|
|
51
|
+
# sync sensor in core/defs/) are plain @dg.definitions functions, not a
|
|
52
|
+
# Component, so they load via load_from_defs_folder — which works even from
|
|
53
|
+
# the wheel because core force-includes its own pyproject.toml.
|
|
54
|
+
core_module = import_module("core")
|
|
55
|
+
package_file = core_module.__file__
|
|
56
|
+
if package_file is None:
|
|
57
|
+
raise RuntimeError(
|
|
58
|
+
"Cannot resolve core's package root: core.__file__ is missing."
|
|
59
|
+
)
|
|
60
|
+
return load_from_defs_folder(
|
|
61
|
+
path_within_project=Path(package_file).resolve().parent
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _load_plugin_defs() -> list[Definitions]:
|
|
66
|
+
# Discover plugins from installed dist metadata (entry points), NOT by walking
|
|
67
|
+
# the filesystem for a Dagster-project layout. load_from_defs_folder needs an
|
|
68
|
+
# on-disk [tool.dg] pyproject that uv_build wheels do not ship adjacent in
|
|
69
|
+
# site-packages, so it silently returned 0 assets from installed wheels.
|
|
70
|
+
# build_defs_for_component builds a standalone ComponentLoadContext and is
|
|
71
|
+
# install-mode-independent. Do not revert plugins to load_from_defs_folder.
|
|
72
|
+
discovered = sorted(
|
|
73
|
+
entry_points(group=PLUGIN_ENTRY_POINT_GROUP),
|
|
74
|
+
key=lambda entry_point: entry_point.name,
|
|
75
|
+
)
|
|
76
|
+
return [
|
|
77
|
+
build_defs_for_component(entry_point.load()()) for entry_point in discovered
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _build_base_platform_client() -> BasePlatformClient:
|
|
82
|
+
settings = load_shared_python_settings()
|
|
83
|
+
if settings.ctx_platform_url is None:
|
|
84
|
+
raise RuntimeError(
|
|
85
|
+
f"{CTX_PLATFORM_URL_ENV_VAR} is not set; run via ctxb so the "
|
|
86
|
+
"platform URL is configured."
|
|
87
|
+
)
|
|
88
|
+
if settings.ctx_base_id is None:
|
|
89
|
+
raise RuntimeError(
|
|
90
|
+
f"{CTXB_BASE_ID_ENV_VAR} is not set; run via ctxb so the base id is "
|
|
91
|
+
"configured."
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
# resolve_machine_token_auth() is the presence-switch: machine creds set
|
|
95
|
+
# (platform-managed) -> every platform call carries the minted aud=ctx
|
|
96
|
+
# Bearer; none set (local dev) -> tokenless, the platform authorizes via
|
|
97
|
+
# its CTX_AUTH_SUB_ID trust-env mode; partial -> raises here, at run
|
|
98
|
+
# launch, like the URL/base-id guards above.
|
|
99
|
+
return BasePlatformClient(
|
|
100
|
+
platform_url=settings.ctx_platform_url,
|
|
101
|
+
base_id=settings.ctx_base_id,
|
|
102
|
+
auth=resolve_machine_token_auth(),
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _build_provider_token_client() -> ProviderTokenClient:
|
|
107
|
+
# ProviderTokenSettings() reads the host-injected machine creds (CTX_WEB_URL /
|
|
108
|
+
# CTX_MACHINE_CLIENT_ID / CTX_MACHINE_CLIENT_SECRET) plus CTXB_SCRATCH_DIR
|
|
109
|
+
# (the deployment-shared token cache) and fails loud if any is missing or
|
|
110
|
+
# blank. Registered lazily below (twin of base_platform) so a read-only
|
|
111
|
+
# introspection load never demands the machine creds it won't use; the ~1h
|
|
112
|
+
# machine token lives in the shared cache, so redemptions reuse the
|
|
113
|
+
# deployment's one token instead of minting per run.
|
|
114
|
+
return ProviderTokenClient(ProviderTokenSettings())
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@definitions
|
|
118
|
+
def defs():
|
|
119
|
+
merged = Definitions.merge(
|
|
120
|
+
_without_component_tree(_load_core_defs()),
|
|
121
|
+
*[_without_component_tree(plugin_defs) for plugin_defs in _load_plugin_defs()],
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
resources = dict(merged.resources or {})
|
|
125
|
+
# Lazy on purpose: building the client eagerly here would construct
|
|
126
|
+
# SharedPythonSettings (which requires CTXB_DATABASE_URL) during code-location
|
|
127
|
+
# load, making read-only introspection (assetNodes/runs) demand a warehouse URL
|
|
128
|
+
# it never uses. resource_fn runs only at execution. Do not revert to
|
|
129
|
+
# hardcoded_resource.
|
|
130
|
+
resources["base_platform"] = ResourceDefinition(
|
|
131
|
+
resource_fn=lambda _init_context: _build_base_platform_client()
|
|
132
|
+
)
|
|
133
|
+
resources["provider_token"] = ResourceDefinition(
|
|
134
|
+
resource_fn=lambda _init_context: _build_provider_token_client()
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
return Definitions(
|
|
138
|
+
assets=merged.assets,
|
|
139
|
+
schedules=merged.schedules,
|
|
140
|
+
sensors=merged.sensors,
|
|
141
|
+
jobs=merged.jobs,
|
|
142
|
+
resources=resources,
|
|
143
|
+
executor=merged.executor,
|
|
144
|
+
loggers=merged.loggers,
|
|
145
|
+
asset_checks=merged.asset_checks,
|
|
146
|
+
metadata=merged.metadata,
|
|
147
|
+
)
|
core/defs/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
|
|
5
|
+
import dagster as dg
|
|
6
|
+
from shared_plugins.base_platform import BasePlatformClient
|
|
7
|
+
from shared_plugins.naming import dagster_partition_def_name
|
|
8
|
+
|
|
9
|
+
PARTITION_SYNC_SENSOR_MIN_INTERVAL_SECONDS = 30
|
|
10
|
+
BINDING_PARTITIONS_DEF_SUFFIX = "_bindings"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dg.sensor(
|
|
14
|
+
name="binding_partitions_sync_sensor",
|
|
15
|
+
minimum_interval_seconds=PARTITION_SYNC_SENSOR_MIN_INTERVAL_SECONDS,
|
|
16
|
+
default_status=dg.DefaultSensorStatus.RUNNING,
|
|
17
|
+
)
|
|
18
|
+
def binding_partitions_sync_sensor(
|
|
19
|
+
context: dg.SensorEvaluationContext,
|
|
20
|
+
base_platform: dg.ResourceParam[BasePlatformClient],
|
|
21
|
+
) -> dg.SensorResult:
|
|
22
|
+
plan = base_platform.load_binding_plan()
|
|
23
|
+
|
|
24
|
+
active_binding_ids_by_partitions_def: dict[str, set[str]] = defaultdict(set)
|
|
25
|
+
for binding in plan.bindings:
|
|
26
|
+
partitions_def_name = dagster_partition_def_name(binding.plugin_id)
|
|
27
|
+
active_binding_ids_by_partitions_def[partitions_def_name].add(
|
|
28
|
+
binding.binding_id
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
known_partitions_defs = set(active_binding_ids_by_partitions_def)
|
|
32
|
+
repository_def = context.repository_def
|
|
33
|
+
if repository_def is not None:
|
|
34
|
+
for asset_key in repository_def.asset_graph.get_all_asset_keys():
|
|
35
|
+
partitions_def = repository_def.asset_graph.get(asset_key).partitions_def
|
|
36
|
+
if isinstance(partitions_def, dg.DynamicPartitionsDefinition):
|
|
37
|
+
if partitions_def.name.endswith(BINDING_PARTITIONS_DEF_SUFFIX):
|
|
38
|
+
known_partitions_defs.add(partitions_def.name)
|
|
39
|
+
|
|
40
|
+
dynamic_partition_requests = []
|
|
41
|
+
|
|
42
|
+
for partitions_def_name in sorted(known_partitions_defs):
|
|
43
|
+
active_binding_ids = active_binding_ids_by_partitions_def.get(
|
|
44
|
+
partitions_def_name, set()
|
|
45
|
+
)
|
|
46
|
+
partitions_def = dg.DynamicPartitionsDefinition(name=partitions_def_name)
|
|
47
|
+
existing_partition_ids = set(
|
|
48
|
+
context.instance.get_dynamic_partitions(partitions_def_name)
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
partition_additions = sorted(active_binding_ids - existing_partition_ids)
|
|
52
|
+
partition_deletions = sorted(existing_partition_ids - active_binding_ids)
|
|
53
|
+
|
|
54
|
+
if partition_additions:
|
|
55
|
+
dynamic_partition_requests.append(
|
|
56
|
+
partitions_def.build_add_request(partition_additions)
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
if partition_deletions:
|
|
60
|
+
dynamic_partition_requests.append(
|
|
61
|
+
partitions_def.build_delete_request(partition_deletions)
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
if not dynamic_partition_requests:
|
|
65
|
+
return dg.SensorResult(skip_reason="No dynamic partition key changes.")
|
|
66
|
+
|
|
67
|
+
return dg.SensorResult(dynamic_partitions_requests=dynamic_partition_requests)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dg.definitions
|
|
71
|
+
def defs() -> dg.Definitions:
|
|
72
|
+
return dg.Definitions(sensors=[binding_partitions_sync_sensor])
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Sequence
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
import dagster as dg
|
|
9
|
+
from base_client.sync_state import SyncStateInputBindingsValue
|
|
10
|
+
from shared_plugins.base_platform import BasePlatformClient
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from dagster._core.definitions.assets.graph.base_asset_graph import BaseAssetGraph
|
|
14
|
+
|
|
15
|
+
# A binding's frontier/last_run are projected from the runs touching its plugin's
|
|
16
|
+
# canonical (ctx_primary) sync assets, then reported up via base_platform.
|
|
17
|
+
CTX_PRIMARY_TAG_KEY = "ctx_primary"
|
|
18
|
+
CTX_PRIMARY_TAG_VALUE = "true"
|
|
19
|
+
# Dagster's per-run partition tag; a binding-sync run carries its binding_id here.
|
|
20
|
+
PARTITION_TAG_KEY = "dagster/partition"
|
|
21
|
+
RUNS_PER_BINDING_LIMIT = 50
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class RunRecord:
|
|
26
|
+
"""One Dagster run projected to the fields the fold needs (mirrors
|
|
27
|
+
state.ts's ``RunRecord``): the canonical asset keys it touched, its start
|
|
28
|
+
time in epoch seconds (``None`` until the run starts), and its status
|
|
29
|
+
surfaced verbatim."""
|
|
30
|
+
|
|
31
|
+
asset_keys: tuple[str, ...]
|
|
32
|
+
start_time: float | None
|
|
33
|
+
status: str
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def fold_binding_sync_state(
|
|
37
|
+
canonical_keys: frozenset[str],
|
|
38
|
+
runs: Sequence[RunRecord],
|
|
39
|
+
) -> SyncStateInputBindingsValue:
|
|
40
|
+
"""Fold one binding's runs (newest-first) against its plugin's canonical
|
|
41
|
+
asset keys into the published digest — a verbatim port of state.ts's
|
|
42
|
+
``foldBindingState``.
|
|
43
|
+
|
|
44
|
+
``frontier`` is conservative: the min, over the binding's *active* canonical
|
|
45
|
+
assets (those seen in the runs), of each asset's latest successful run;
|
|
46
|
+
``None`` if any active asset has no success in view. ``last_run`` /
|
|
47
|
+
``last_run_status`` come from the newest run touching any canonical asset, in
|
|
48
|
+
whatever status it is.
|
|
49
|
+
"""
|
|
50
|
+
relevant = [
|
|
51
|
+
run for run in runs if any(key in canonical_keys for key in run.asset_keys)
|
|
52
|
+
]
|
|
53
|
+
if not relevant:
|
|
54
|
+
return SyncStateInputBindingsValue(
|
|
55
|
+
frontier=None, last_run=None, last_run_status=None
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
latest = relevant[0]
|
|
59
|
+
last_run = None if latest.start_time is None else _epoch_to_utc(latest.start_time)
|
|
60
|
+
last_run_status = latest.status
|
|
61
|
+
|
|
62
|
+
active_keys = {
|
|
63
|
+
key for run in relevant for key in run.asset_keys if key in canonical_keys
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
frontier_seconds: float | None = None
|
|
67
|
+
for key in active_keys:
|
|
68
|
+
success = next(
|
|
69
|
+
(
|
|
70
|
+
run
|
|
71
|
+
for run in relevant
|
|
72
|
+
if run.status == dg.DagsterRunStatus.SUCCESS.value
|
|
73
|
+
and run.start_time is not None
|
|
74
|
+
and key in run.asset_keys
|
|
75
|
+
),
|
|
76
|
+
None,
|
|
77
|
+
)
|
|
78
|
+
if success is None or success.start_time is None:
|
|
79
|
+
return SyncStateInputBindingsValue(
|
|
80
|
+
frontier=None, last_run=last_run, last_run_status=last_run_status
|
|
81
|
+
)
|
|
82
|
+
frontier_seconds = (
|
|
83
|
+
success.start_time
|
|
84
|
+
if frontier_seconds is None
|
|
85
|
+
else min(frontier_seconds, success.start_time)
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
return SyncStateInputBindingsValue(
|
|
89
|
+
frontier=None if frontier_seconds is None else _epoch_to_utc(frontier_seconds),
|
|
90
|
+
last_run=last_run,
|
|
91
|
+
last_run_status=last_run_status,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _epoch_to_utc(seconds: float) -> datetime:
|
|
96
|
+
return datetime.fromtimestamp(seconds, tz=timezone.utc)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _is_canonical(asset_graph: BaseAssetGraph, asset_key: dg.AssetKey) -> bool:
|
|
100
|
+
# A selection key absent from the live graph (its plugin removed/renamed
|
|
101
|
+
# between the run finishing and this sensor firing) is simply not canonical —
|
|
102
|
+
# matching state.ts, which only membership-tests selection keys and never
|
|
103
|
+
# looks them up. A bare asset_graph.get() would raise KeyError, and since
|
|
104
|
+
# run-status sensors share a cursor, a throwing tick would stall digest
|
|
105
|
+
# reporting for every binding, not just this one.
|
|
106
|
+
if not asset_graph.has(asset_key):
|
|
107
|
+
return False
|
|
108
|
+
tags = asset_graph.get(asset_key).tags
|
|
109
|
+
return tags.get(CTX_PRIMARY_TAG_KEY) == CTX_PRIMARY_TAG_VALUE
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def canonical_asset_keys_for_group(
|
|
113
|
+
asset_graph: BaseAssetGraph, group_name: str
|
|
114
|
+
) -> frozenset[str]:
|
|
115
|
+
"""The user-string keys of a plugin group's canonical (``ctx_primary``)
|
|
116
|
+
assets — the in-process replacement for state.ts's
|
|
117
|
+
``extractCanonicalKeysByPlugin`` (which walked the GraphQL ``assetNodes``)."""
|
|
118
|
+
return frozenset(
|
|
119
|
+
asset_key.to_user_string()
|
|
120
|
+
for asset_key in asset_graph.asset_keys_for_group(group_name)
|
|
121
|
+
if _is_canonical(asset_graph, asset_key)
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _binding_run_records(
|
|
126
|
+
instance: dg.DagsterInstance, binding_id: str
|
|
127
|
+
) -> list[RunRecord]:
|
|
128
|
+
"""The binding's most-recent runs (newest-first, capped) projected to
|
|
129
|
+
``RunRecord`` — the native-instance replacement for state.ts's per-binding
|
|
130
|
+
``runsOrError`` GraphQL query."""
|
|
131
|
+
# No explicit order_by: get_run_records then sorts by the runs table's
|
|
132
|
+
# autoincrement id DESC — a strict total order (newest-created first) with no
|
|
133
|
+
# ties. create_timestamp DESC would tie when runs are created in the same
|
|
134
|
+
# instant, making "latest" (and the frontier's per-key newest-success pick) flap.
|
|
135
|
+
records = instance.get_run_records(
|
|
136
|
+
filters=dg.RunsFilter(tags={PARTITION_TAG_KEY: binding_id}),
|
|
137
|
+
limit=RUNS_PER_BINDING_LIMIT,
|
|
138
|
+
)
|
|
139
|
+
return [
|
|
140
|
+
RunRecord(
|
|
141
|
+
asset_keys=tuple(
|
|
142
|
+
asset_key.to_user_string()
|
|
143
|
+
for asset_key in (record.dagster_run.asset_selection or ())
|
|
144
|
+
),
|
|
145
|
+
start_time=record.start_time,
|
|
146
|
+
status=record.dagster_run.status.value,
|
|
147
|
+
)
|
|
148
|
+
for record in records
|
|
149
|
+
]
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _report_binding_sync_state_for_run(
|
|
153
|
+
context: dg.RunStatusSensorContext,
|
|
154
|
+
base_platform: BasePlatformClient,
|
|
155
|
+
) -> None:
|
|
156
|
+
"""Recompute and push one binding's digest in response to a run-status
|
|
157
|
+
transition. Skips runs that are not a canonical binding sync — no partition
|
|
158
|
+
key, or no ``ctx_primary`` asset in the run's selection (the binding's plugin
|
|
159
|
+
group is read off that selection)."""
|
|
160
|
+
binding_id = context.partition_key
|
|
161
|
+
if binding_id is None:
|
|
162
|
+
return
|
|
163
|
+
|
|
164
|
+
asset_graph = context.repository_def.asset_graph
|
|
165
|
+
selection = context.dagster_run.asset_selection or frozenset()
|
|
166
|
+
group_name = next(
|
|
167
|
+
(
|
|
168
|
+
asset_graph.get(asset_key).group_name
|
|
169
|
+
for asset_key in selection
|
|
170
|
+
if _is_canonical(asset_graph, asset_key)
|
|
171
|
+
),
|
|
172
|
+
None,
|
|
173
|
+
)
|
|
174
|
+
if group_name is None:
|
|
175
|
+
return
|
|
176
|
+
|
|
177
|
+
canonical_keys = canonical_asset_keys_for_group(asset_graph, group_name)
|
|
178
|
+
runs = _binding_run_records(context.instance, binding_id)
|
|
179
|
+
state = fold_binding_sync_state(canonical_keys, runs)
|
|
180
|
+
base_platform.report_binding_sync_state(binding_id, state)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
# Two sensors, one producer: ``@run_status_sensor`` monitors a single status, so
|
|
184
|
+
# SUCCESS (advances the frontier) and FAILURE (surfaces a broken binding) each
|
|
185
|
+
# get a thin wrapper delegating to the shared producer above. STARTED/CANCELED
|
|
186
|
+
# are deferred — a binding's transient "syncing"/"canceled" state is not needed
|
|
187
|
+
# by the freshness/health view.
|
|
188
|
+
@dg.run_status_sensor(
|
|
189
|
+
name="binding_sync_state_success_sensor",
|
|
190
|
+
run_status=dg.DagsterRunStatus.SUCCESS,
|
|
191
|
+
default_status=dg.DefaultSensorStatus.RUNNING,
|
|
192
|
+
minimum_interval_seconds=30,
|
|
193
|
+
)
|
|
194
|
+
def binding_sync_state_success_sensor(
|
|
195
|
+
context: dg.RunStatusSensorContext,
|
|
196
|
+
base_platform: dg.ResourceParam[BasePlatformClient],
|
|
197
|
+
) -> None:
|
|
198
|
+
_report_binding_sync_state_for_run(context, base_platform)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@dg.run_status_sensor(
|
|
202
|
+
name="binding_sync_state_failure_sensor",
|
|
203
|
+
run_status=dg.DagsterRunStatus.FAILURE,
|
|
204
|
+
default_status=dg.DefaultSensorStatus.RUNNING,
|
|
205
|
+
minimum_interval_seconds=30,
|
|
206
|
+
)
|
|
207
|
+
def binding_sync_state_failure_sensor(
|
|
208
|
+
context: dg.RunStatusSensorContext,
|
|
209
|
+
base_platform: dg.ResourceParam[BasePlatformClient],
|
|
210
|
+
) -> None:
|
|
211
|
+
_report_binding_sync_state_for_run(context, base_platform)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
@dg.definitions
|
|
215
|
+
def defs() -> dg.Definitions:
|
|
216
|
+
return dg.Definitions(
|
|
217
|
+
sensors=[
|
|
218
|
+
binding_sync_state_success_sensor,
|
|
219
|
+
binding_sync_state_failure_sensor,
|
|
220
|
+
]
|
|
221
|
+
)
|
pyproject.toml
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "contextbase-core"
|
|
3
|
+
requires-python = ">=3.12,<3.13"
|
|
4
|
+
version = "0.0.0-alpha-1"
|
|
5
|
+
dependencies = [
|
|
6
|
+
"contextbase-plugin-asana==0.0.0-alpha-1",
|
|
7
|
+
"contextbase-plugin-chrome-local==0.0.0-alpha-1",
|
|
8
|
+
"contextbase-plugin-claude-local==0.0.0-alpha-1",
|
|
9
|
+
"contextbase-plugin-clickup==0.0.0-alpha-1",
|
|
10
|
+
"contextbase-plugin-codex-local==0.0.0-alpha-1",
|
|
11
|
+
"contextbase-plugin-github==0.0.0-alpha-1",
|
|
12
|
+
"contextbase-plugin-gmail==0.0.0-alpha-1",
|
|
13
|
+
"contextbase-plugin-gong==0.0.0-alpha-1",
|
|
14
|
+
"contextbase-plugin-google-calendar==0.0.0-alpha-1",
|
|
15
|
+
"contextbase-plugin-granola==0.0.0-alpha-1",
|
|
16
|
+
"contextbase-plugin-hubspot==0.0.0-alpha-1",
|
|
17
|
+
"contextbase-plugin-icloud-tabs-local==0.0.0-alpha-1",
|
|
18
|
+
"contextbase-plugin-imessage-local==0.0.0-alpha-1",
|
|
19
|
+
"contextbase-plugin-intercom==0.0.0-alpha-1",
|
|
20
|
+
"contextbase-plugin-jira==0.0.0-alpha-1",
|
|
21
|
+
"contextbase-plugin-linear==0.0.0-alpha-1",
|
|
22
|
+
"contextbase-plugin-microsoft-calendar==0.0.0-alpha-1",
|
|
23
|
+
"contextbase-plugin-microsoft-mail==0.0.0-alpha-1",
|
|
24
|
+
"contextbase-plugin-microsoft-planner-premium==0.0.0-alpha-1",
|
|
25
|
+
"contextbase-plugin-notion==0.0.0-alpha-1",
|
|
26
|
+
"contextbase-plugin-reminders-local==0.0.0-alpha-1",
|
|
27
|
+
"contextbase-plugin-salesforce==0.0.0-alpha-1",
|
|
28
|
+
"contextbase-plugin-slack==0.0.0-alpha-1",
|
|
29
|
+
"contextbase-plugin-whatsapp-local==0.0.0-alpha-1",
|
|
30
|
+
"contextbase-plugin-workflowy==0.0.0-alpha-1",
|
|
31
|
+
"contextbase-shared-plugins==0.0.0-alpha-1",
|
|
32
|
+
"dagster==1.12.14",
|
|
33
|
+
"httpx>=0.28.1",
|
|
34
|
+
"pydantic-settings>=2.11.0",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[dependency-groups]
|
|
38
|
+
dev = [
|
|
39
|
+
"dagster-webserver",
|
|
40
|
+
"dagster-dg-cli",
|
|
41
|
+
"pytest>=8.4.2",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[build-system]
|
|
45
|
+
requires = ["hatchling"]
|
|
46
|
+
build-backend = "hatchling.build"
|
|
47
|
+
|
|
48
|
+
[tool.hatch.build.targets.wheel]
|
|
49
|
+
packages = ["src/core"]
|
|
50
|
+
force-include = { "pyproject.toml" = "pyproject.toml" }
|
|
51
|
+
|
|
52
|
+
[tool.dg]
|
|
53
|
+
directory_type = "project"
|
|
54
|
+
|
|
55
|
+
[tool.dg.project]
|
|
56
|
+
root_module = "core"
|
|
57
|
+
|
|
58
|
+
[tool.uv.sources]
|
|
59
|
+
contextbase-plugin-asana = { workspace = true }
|
|
60
|
+
contextbase-plugin-chrome-local = { workspace = true }
|
|
61
|
+
contextbase-plugin-claude-local = { workspace = true }
|
|
62
|
+
contextbase-plugin-clickup = { workspace = true }
|
|
63
|
+
contextbase-plugin-codex-local = { workspace = true }
|
|
64
|
+
contextbase-plugin-github = { workspace = true }
|
|
65
|
+
contextbase-plugin-gmail = { workspace = true }
|
|
66
|
+
contextbase-plugin-gong = { workspace = true }
|
|
67
|
+
contextbase-plugin-google-calendar = { workspace = true }
|
|
68
|
+
contextbase-plugin-granola = { workspace = true }
|
|
69
|
+
contextbase-plugin-hubspot = { workspace = true }
|
|
70
|
+
contextbase-plugin-icloud-tabs-local = { workspace = true }
|
|
71
|
+
contextbase-plugin-imessage-local = { workspace = true }
|
|
72
|
+
contextbase-plugin-intercom = { workspace = true }
|
|
73
|
+
contextbase-plugin-jira = { workspace = true }
|
|
74
|
+
contextbase-plugin-linear = { workspace = true }
|
|
75
|
+
contextbase-plugin-microsoft-calendar = { workspace = true }
|
|
76
|
+
contextbase-plugin-microsoft-mail = { workspace = true }
|
|
77
|
+
contextbase-plugin-microsoft-planner-premium = { workspace = true }
|
|
78
|
+
contextbase-plugin-notion = { workspace = true }
|
|
79
|
+
contextbase-plugin-reminders-local = { workspace = true }
|
|
80
|
+
contextbase-plugin-salesforce = { workspace = true }
|
|
81
|
+
contextbase-plugin-slack = { workspace = true }
|
|
82
|
+
contextbase-plugin-whatsapp-local = { workspace = true }
|
|
83
|
+
contextbase-plugin-workflowy = { workspace = true }
|
|
84
|
+
contextbase-shared-plugins = { workspace = true }
|