agentgraph-server 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentgraph/__init__.py +1 -0
- agentgraph/auth/__init__.py +0 -0
- agentgraph/auth/credentials.py +224 -0
- agentgraph/backends/__init__.py +50 -0
- agentgraph/backends/sqlite/__init__.py +1 -0
- agentgraph/backends/sqlite/backend.py +1471 -0
- agentgraph/backends/sqlite/vector.py +142 -0
- agentgraph/cli.py +721 -0
- agentgraph/cli_query.py +519 -0
- agentgraph/config.py +90 -0
- agentgraph/connectors/__init__.py +0 -0
- agentgraph/connectors/base.py +455 -0
- agentgraph/connectors/registry.py +78 -0
- agentgraph/connectors/status.py +244 -0
- agentgraph/core/__init__.py +0 -0
- agentgraph/core/context.py +26 -0
- agentgraph/core/runtime.py +36 -0
- agentgraph/core/storage.py +240 -0
- agentgraph/graph/__init__.py +1 -0
- agentgraph/graph/bookmark.py +87 -0
- agentgraph/graph/delete.py +17 -0
- agentgraph/graph/download.py +35 -0
- agentgraph/graph/embeddings.py +58 -0
- agentgraph/graph/fetch.py +53 -0
- agentgraph/graph/gc.py +26 -0
- agentgraph/graph/link.py +63 -0
- agentgraph/graph/person.py +40 -0
- agentgraph/graph/query.py +244 -0
- agentgraph/graph/upsert.py +49 -0
- agentgraph/logging.py +78 -0
- agentgraph/mcp/__init__.py +0 -0
- agentgraph/mcp/server.py +811 -0
- agentgraph/perf.py +43 -0
- agentgraph/server/__init__.py +0 -0
- agentgraph/server/app.py +133 -0
- agentgraph/server/cli_api.py +708 -0
- agentgraph/server/dwell.py +79 -0
- agentgraph/server/graph_api.py +46 -0
- agentgraph/server/router.py +47 -0
- agentgraph/server/sync.py +247 -0
- agentgraph/skills.py +93 -0
- agentgraph_server-0.5.0.data/data/.agents/skills/graph/SKILL.md +159 -0
- agentgraph_server-0.5.0.data/data/.agents/skills/slack-auth/SKILL.md +92 -0
- agentgraph_server-0.5.0.dist-info/METADATA +286 -0
- agentgraph_server-0.5.0.dist-info/RECORD +49 -0
- agentgraph_server-0.5.0.dist-info/WHEEL +5 -0
- agentgraph_server-0.5.0.dist-info/entry_points.txt +2 -0
- agentgraph_server-0.5.0.dist-info/licenses/LICENSE +21 -0
- agentgraph_server-0.5.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Dwell dispatch: classifies a URL and fires a connector fetch."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from agentgraph.connectors.base import ResourceType
|
|
10
|
+
from agentgraph.server.router import classify_observation_url
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
async def record_dwell_time(url: str, dwell_ms: int, meta: dict[str, str] | None = None) -> dict[str, Any]:
|
|
16
|
+
"""Classify url and increment its cumulative dwell time in the backend."""
|
|
17
|
+
ref = await classify_observation_url(url)
|
|
18
|
+
if ref is None:
|
|
19
|
+
logger.debug("report-dwell: unrecognised URL %s", url)
|
|
20
|
+
return {"status": "ignored", "reason": "unrecognised URL"}
|
|
21
|
+
|
|
22
|
+
from agentgraph.config import get_settings
|
|
23
|
+
from agentgraph.core.context import get_backend
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
backend = get_backend()
|
|
27
|
+
await backend.increment_dwell_time(ref.source, ref.resource_id, dwell_ms)
|
|
28
|
+
logger.debug(
|
|
29
|
+
"Recorded dwell time: +%dms for %s %s/%s",
|
|
30
|
+
dwell_ms, ref.source, ref.resource_type, ref.resource_id
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
# Dispatch background connector fetch if the dwell time meets the threshold
|
|
34
|
+
threshold_ms = get_settings().dwell_threshold_seconds * 1000
|
|
35
|
+
if dwell_ms >= threshold_ms:
|
|
36
|
+
logger.info(
|
|
37
|
+
"Dwell threshold met (%dms >= %dms): dispatching fetch for %s %s/%s",
|
|
38
|
+
dwell_ms, threshold_ms, ref.source, ref.resource_type, ref.resource_id
|
|
39
|
+
)
|
|
40
|
+
fetch_meta = dict(meta or {})
|
|
41
|
+
fetch_meta.update(ref.fetch_meta or {})
|
|
42
|
+
asyncio.create_task(
|
|
43
|
+
_dispatch(ref.source, ref.resource_type, ref.resource_id, fetch_meta or None)
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
return {"status": "accepted", "source": ref.source, "resource_type": ref.resource_type}
|
|
47
|
+
except Exception:
|
|
48
|
+
logger.exception("Failed to record dwell time for %s", url)
|
|
49
|
+
return {"status": "error", "reason": "internal backend error"}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
async def _dispatch(
|
|
53
|
+
source: str,
|
|
54
|
+
resource_type: ResourceType,
|
|
55
|
+
resource_id: str,
|
|
56
|
+
meta: dict[str, str] | None = None,
|
|
57
|
+
) -> None:
|
|
58
|
+
from agentgraph.connectors.registry import get_connector
|
|
59
|
+
|
|
60
|
+
connector = get_connector(source)
|
|
61
|
+
if connector is None:
|
|
62
|
+
logger.debug("No connector registered for source '%s'", source)
|
|
63
|
+
return
|
|
64
|
+
try:
|
|
65
|
+
logger.info("Fetching %s/%s/%s", source, resource_type, resource_id)
|
|
66
|
+
batch = await connector.fetch(
|
|
67
|
+
resource_type=resource_type, resource_id=resource_id, meta=meta
|
|
68
|
+
)
|
|
69
|
+
if batch.entities or batch.persons or batch.edges:
|
|
70
|
+
from agentgraph.graph.upsert import upsert_batch
|
|
71
|
+
|
|
72
|
+
await upsert_batch(batch)
|
|
73
|
+
logger.info(
|
|
74
|
+
"Fetch complete %s/%s/%s — %d entities, %d persons, %d edges",
|
|
75
|
+
source, resource_type, resource_id,
|
|
76
|
+
len(batch.entities), len(batch.persons), len(batch.edges),
|
|
77
|
+
)
|
|
78
|
+
except Exception:
|
|
79
|
+
logger.exception("Connector fetch failed: %s/%s/%s", source, resource_type, resource_id)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Graph API router — admin endpoints."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from fastapi import APIRouter
|
|
8
|
+
|
|
9
|
+
router = APIRouter(prefix="/api", tags=["graph"])
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@router.post("/admin/relink")
|
|
13
|
+
async def relink_all() -> dict[str, Any]:
|
|
14
|
+
"""Re-run URL reference linking for all entities that have content.
|
|
15
|
+
|
|
16
|
+
Creates stub entities for any classifiable URLs not yet in the graph.
|
|
17
|
+
Safe to call multiple times — uses ON CONFLICT DO NOTHING/UPDATE.
|
|
18
|
+
"""
|
|
19
|
+
from agentgraph.core.context import get_backend
|
|
20
|
+
from agentgraph.graph.link import link_entity_to_urls
|
|
21
|
+
|
|
22
|
+
rows = await get_backend().list_entities(
|
|
23
|
+
entity_types=None,
|
|
24
|
+
platform=None,
|
|
25
|
+
since=None,
|
|
26
|
+
limit=100_000,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
total_links = 0
|
|
30
|
+
for row in rows:
|
|
31
|
+
content = row.get("content")
|
|
32
|
+
platform = row.get("platform")
|
|
33
|
+
platform_entity_id = row.get("platform_entity_id")
|
|
34
|
+
if not (
|
|
35
|
+
isinstance(content, str)
|
|
36
|
+
and isinstance(platform, str)
|
|
37
|
+
and isinstance(platform_entity_id, str)
|
|
38
|
+
):
|
|
39
|
+
continue
|
|
40
|
+
total_links += await link_entity_to_urls(
|
|
41
|
+
platform_entity_id,
|
|
42
|
+
platform,
|
|
43
|
+
content,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
return {"relinked": len(rows), "edges_created": total_links}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""URL routing through connector-owned resolvers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from agentgraph.connectors.base import SourceReference
|
|
8
|
+
|
|
9
|
+
# U+200B zero-width space, U+200C/D/E/F directional marks, U+00AD soft hyphen, U+FEFF BOM
|
|
10
|
+
_INVISIBLE_CHARS_RE = re.compile(
|
|
11
|
+
"[]+"
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def normalise_url_for_matching(url: str) -> str:
|
|
16
|
+
"""Strip invisible characters and punctuation commonly wrapping URLs in prose."""
|
|
17
|
+
return _INVISIBLE_CHARS_RE.sub("", url).rstrip(".,)>\"'")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def classify_url(url: str) -> SourceReference | None:
|
|
21
|
+
"""Return a SourceReference for a connector-owned URL, or None."""
|
|
22
|
+
from agentgraph.connectors.registry import bootstrap, get_all_connectors
|
|
23
|
+
|
|
24
|
+
normalised_url = normalise_url_for_matching(url)
|
|
25
|
+
bootstrap()
|
|
26
|
+
for connector in get_all_connectors():
|
|
27
|
+
if type(connector).is_generic_url_fallback:
|
|
28
|
+
continue
|
|
29
|
+
ref = connector.resolve_url(normalised_url)
|
|
30
|
+
if ref is not None:
|
|
31
|
+
return ref
|
|
32
|
+
return None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def classify_observation_url(url: str) -> SourceReference | None:
|
|
36
|
+
"""Resolve a browser-observed URL through connector-owned async resolvers."""
|
|
37
|
+
from agentgraph.connectors.registry import bootstrap, get_all_connectors
|
|
38
|
+
|
|
39
|
+
normalised_url = normalise_url_for_matching(url)
|
|
40
|
+
bootstrap()
|
|
41
|
+
for connector in get_all_connectors():
|
|
42
|
+
if type(connector).is_generic_url_fallback:
|
|
43
|
+
continue
|
|
44
|
+
ref = await connector.resolve_observation_url(normalised_url)
|
|
45
|
+
if ref is not None:
|
|
46
|
+
return ref
|
|
47
|
+
return None
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""SyncEngine: background polling for all connectors with poll_interval set."""
|
|
2
|
+
|
|
3
|
+
# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false
|
|
4
|
+
# pyright: reportUnknownArgumentType=false
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
import logging
|
|
10
|
+
from datetime import UTC, datetime, timedelta
|
|
11
|
+
from time import perf_counter
|
|
12
|
+
from typing import Literal, TypedDict, cast
|
|
13
|
+
|
|
14
|
+
from apscheduler.schedulers.asyncio import AsyncIOScheduler # type: ignore[import-untyped]
|
|
15
|
+
|
|
16
|
+
from agentgraph.connectors.base import BaseConnector
|
|
17
|
+
from agentgraph.connectors.status import connector_uses_auth
|
|
18
|
+
from agentgraph.core.context import get_backend
|
|
19
|
+
from agentgraph.graph.upsert import upsert_batch
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
_failure_counts: dict[str, int] = {}
|
|
24
|
+
_backoff_until: dict[str, datetime] = {}
|
|
25
|
+
_manual_poll_tasks: dict[str, asyncio.Task[None]] = {}
|
|
26
|
+
_active_poll_tasks: dict[str, set[asyncio.Task[None]]] = {}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
PollScheduleStatus = Literal["queued", "already_running", "skipped"]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class PollScheduleResult(TypedDict):
|
|
33
|
+
source: str
|
|
34
|
+
status: PollScheduleStatus
|
|
35
|
+
reason: str | None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def clear_poll_backoff() -> None:
|
|
39
|
+
_failure_counts.clear()
|
|
40
|
+
_backoff_until.clear()
|
|
41
|
+
_manual_poll_tasks.clear()
|
|
42
|
+
_active_poll_tasks.clear()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
async def shutdown_poll_tasks(*, timeout: float = 10.0) -> None:
|
|
46
|
+
"""Cancel in-flight poll tasks and wait briefly for their cleanup handlers."""
|
|
47
|
+
active_tasks = [task for tasks in _active_poll_tasks.values() for task in tasks]
|
|
48
|
+
tasks = {task for task in [*_manual_poll_tasks.values(), *active_tasks] if not task.done()}
|
|
49
|
+
if not tasks:
|
|
50
|
+
_manual_poll_tasks.clear()
|
|
51
|
+
_active_poll_tasks.clear()
|
|
52
|
+
return
|
|
53
|
+
|
|
54
|
+
for task in tasks:
|
|
55
|
+
task.cancel()
|
|
56
|
+
try:
|
|
57
|
+
await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=timeout)
|
|
58
|
+
except TimeoutError:
|
|
59
|
+
logger.warning("timed out waiting for %d poll task(s) to stop", len(tasks))
|
|
60
|
+
finally:
|
|
61
|
+
_manual_poll_tasks.clear()
|
|
62
|
+
_active_poll_tasks.clear()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def schedule_poll_connector(connector: BaseConnector) -> PollScheduleResult:
|
|
66
|
+
"""Start a manual background poll unless one is already running."""
|
|
67
|
+
source = connector.source
|
|
68
|
+
existing = _manual_poll_tasks.get(source)
|
|
69
|
+
if existing is not None and not existing.done():
|
|
70
|
+
logger.info("poll %s — manual trigger skipped because a poll is already running", source)
|
|
71
|
+
return {"source": source, "status": "already_running", "reason": None}
|
|
72
|
+
|
|
73
|
+
auth_skip_reason = await _auth_skip_reason(connector)
|
|
74
|
+
if auth_skip_reason is not None:
|
|
75
|
+
logger.info("poll %s — manual trigger skipped: %s", source, auth_skip_reason)
|
|
76
|
+
return {"source": source, "status": "skipped", "reason": auth_skip_reason}
|
|
77
|
+
|
|
78
|
+
task = asyncio.create_task(poll_connector(connector))
|
|
79
|
+
_manual_poll_tasks[source] = task
|
|
80
|
+
task.add_done_callback(lambda done_task: _manual_poll_tasks.pop(source, None))
|
|
81
|
+
return {"source": source, "status": "queued", "reason": None}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
async def _auth_skip_reason(connector: BaseConnector) -> str | None:
|
|
85
|
+
if not connector_uses_auth(connector):
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
account_ids = connector.poll_account_ids()
|
|
90
|
+
statuses = [await type(connector).verify_auth(account_id) for account_id in account_ids]
|
|
91
|
+
except Exception as exc:
|
|
92
|
+
return f"authentication check failed: {type(exc).__name__}"
|
|
93
|
+
|
|
94
|
+
invalid = next(((status, detail) for status, detail in statuses if status == "invalid"), None)
|
|
95
|
+
if invalid is not None:
|
|
96
|
+
detail = invalid[1]
|
|
97
|
+
return f"authentication invalid: {detail}" if detail else "authentication invalid"
|
|
98
|
+
|
|
99
|
+
if not any(status == "ok" for status, _ in statuses):
|
|
100
|
+
return "authentication missing"
|
|
101
|
+
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _sync_scope(source: str, account_id: str | None) -> str:
|
|
106
|
+
return source if account_id is None else f"{source}:{account_id}"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _has_local_auth(connector: BaseConnector) -> bool:
|
|
110
|
+
if not connector_uses_auth(connector):
|
|
111
|
+
return True
|
|
112
|
+
accounts = type(connector).list_accounts()
|
|
113
|
+
if accounts:
|
|
114
|
+
return True
|
|
115
|
+
return type(connector).get_authenticated_user() is not None
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _backoff_remaining(source: str) -> timedelta | None:
|
|
119
|
+
until = _backoff_until.get(source)
|
|
120
|
+
if until is None:
|
|
121
|
+
return None
|
|
122
|
+
remaining = until - datetime.now(UTC)
|
|
123
|
+
if remaining <= timedelta(0):
|
|
124
|
+
_backoff_until.pop(source, None)
|
|
125
|
+
return None
|
|
126
|
+
return remaining
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _record_success(source: str) -> None:
|
|
130
|
+
_failure_counts.pop(source, None)
|
|
131
|
+
_backoff_until.pop(source, None)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _record_failure(source: str) -> None:
|
|
135
|
+
count = _failure_counts.get(source, 0) + 1
|
|
136
|
+
_failure_counts[source] = count
|
|
137
|
+
delay = min(60 * (2 ** (count - 1)), 3600)
|
|
138
|
+
_backoff_until[source] = datetime.now(UTC) + timedelta(seconds=delay)
|
|
139
|
+
logger.warning("poll %s — backing off for %ds after %d failure(s)", source, delay, count)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
async def poll_connector(connector: BaseConnector) -> None:
|
|
143
|
+
source = connector.source
|
|
144
|
+
task = cast(asyncio.Task[None] | None, asyncio.current_task())
|
|
145
|
+
if task is not None:
|
|
146
|
+
_active_poll_tasks.setdefault(source, set()).add(task)
|
|
147
|
+
started = perf_counter()
|
|
148
|
+
try:
|
|
149
|
+
remaining = _backoff_remaining(source)
|
|
150
|
+
if remaining is not None:
|
|
151
|
+
logger.info("poll %s — skipped during failure backoff (%.0fs remaining)", source, remaining.total_seconds())
|
|
152
|
+
return
|
|
153
|
+
if not _has_local_auth(connector):
|
|
154
|
+
logger.info("poll %s — skipped because authentication is not configured", source)
|
|
155
|
+
return
|
|
156
|
+
backend = get_backend()
|
|
157
|
+
for account_id in connector.poll_account_ids():
|
|
158
|
+
scope_started = perf_counter()
|
|
159
|
+
scope = _sync_scope(source, account_id)
|
|
160
|
+
cursor = await backend.load_cursor(scope)
|
|
161
|
+
is_first_run = not cursor
|
|
162
|
+
logger.info(
|
|
163
|
+
"poll %s — starting%s",
|
|
164
|
+
scope,
|
|
165
|
+
" (first run / bulk ingest)" if is_first_run else "",
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
batch, new_cursor = await connector.poll(cursor, account_id=account_id)
|
|
169
|
+
|
|
170
|
+
n_entities = len(batch.entities)
|
|
171
|
+
n_persons = len(batch.persons)
|
|
172
|
+
n_edges = len(batch.edges)
|
|
173
|
+
|
|
174
|
+
if batch.entities or batch.persons or batch.edges:
|
|
175
|
+
logger.info(
|
|
176
|
+
"poll %s — upserting %d entities, %d persons, %d edges",
|
|
177
|
+
scope, n_entities, n_persons, n_edges,
|
|
178
|
+
)
|
|
179
|
+
await upsert_batch(batch)
|
|
180
|
+
logger.info("poll %s — upsert complete", scope)
|
|
181
|
+
else:
|
|
182
|
+
logger.info("poll %s — no new data", scope)
|
|
183
|
+
|
|
184
|
+
await backend.save_cursor(scope, new_cursor)
|
|
185
|
+
logger.info("poll %s — completed in %.1fs", scope, perf_counter() - scope_started)
|
|
186
|
+
_record_success(source)
|
|
187
|
+
except Exception:
|
|
188
|
+
_record_failure(source)
|
|
189
|
+
logger.exception("poll failed for connector %s", source)
|
|
190
|
+
finally:
|
|
191
|
+
if task is not None:
|
|
192
|
+
tasks = _active_poll_tasks.get(source)
|
|
193
|
+
if tasks is not None:
|
|
194
|
+
tasks.discard(task)
|
|
195
|
+
if not tasks:
|
|
196
|
+
_active_poll_tasks.pop(source, None)
|
|
197
|
+
logger.debug("poll %s total elapsed %.1fs", source, perf_counter() - started)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
async def run_ingest(connector: BaseConnector) -> None:
|
|
201
|
+
source = connector.source
|
|
202
|
+
started = perf_counter()
|
|
203
|
+
try:
|
|
204
|
+
for account_id in connector.poll_account_ids():
|
|
205
|
+
scope_started = perf_counter()
|
|
206
|
+
scope = _sync_scope(source, account_id)
|
|
207
|
+
logger.info("ingest %s — starting", scope)
|
|
208
|
+
batch = await connector.ingest(account_id=account_id)
|
|
209
|
+
if batch.entities or batch.persons or batch.edges:
|
|
210
|
+
logger.info(
|
|
211
|
+
"ingest %s — upserting %d entities, %d persons, %d edges",
|
|
212
|
+
scope, len(batch.entities), len(batch.persons), len(batch.edges),
|
|
213
|
+
)
|
|
214
|
+
await upsert_batch(batch)
|
|
215
|
+
logger.info("ingest %s — complete in %.1fs", scope, perf_counter() - scope_started)
|
|
216
|
+
else:
|
|
217
|
+
logger.info("ingest %s — no data returned", scope)
|
|
218
|
+
except Exception:
|
|
219
|
+
logger.exception("ingest failed for connector %s", source)
|
|
220
|
+
finally:
|
|
221
|
+
logger.debug("ingest %s total elapsed %.1fs", source, perf_counter() - started)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def setup_sync(scheduler: AsyncIOScheduler) -> None:
|
|
225
|
+
"""Register a poll job for every connector that has poll_interval set."""
|
|
226
|
+
from agentgraph.connectors.registry import get_all_connectors
|
|
227
|
+
|
|
228
|
+
for connector in get_all_connectors():
|
|
229
|
+
interval = connector.poll_interval
|
|
230
|
+
if interval is None:
|
|
231
|
+
continue
|
|
232
|
+
total_seconds = int(interval.total_seconds())
|
|
233
|
+
scheduler.add_job(
|
|
234
|
+
poll_connector,
|
|
235
|
+
"interval",
|
|
236
|
+
seconds=total_seconds,
|
|
237
|
+
args=[connector],
|
|
238
|
+
id=f"sync_{connector.source}",
|
|
239
|
+
name=f"poll connector {connector.source}",
|
|
240
|
+
max_instances=1,
|
|
241
|
+
coalesce=True,
|
|
242
|
+
)
|
|
243
|
+
logger.info(
|
|
244
|
+
"Scheduled background poll for %s every %ds",
|
|
245
|
+
connector.source,
|
|
246
|
+
total_seconds,
|
|
247
|
+
)
|
agentgraph/skills.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Install bundled AgentGraph skills into an agent skill directory."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import shutil
|
|
6
|
+
import sysconfig
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
SkillTarget = Literal["user", "project"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class SkillInstallError(ValueError):
|
|
15
|
+
"""Raised when a skill cannot be installed."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class SkillInstallResult:
|
|
20
|
+
skill: str
|
|
21
|
+
target: SkillTarget
|
|
22
|
+
source: str
|
|
23
|
+
destination: str
|
|
24
|
+
overwritten: bool
|
|
25
|
+
|
|
26
|
+
def to_dict(self) -> dict[str, object]:
|
|
27
|
+
return {
|
|
28
|
+
"skill": self.skill,
|
|
29
|
+
"target": self.target,
|
|
30
|
+
"source": self.source,
|
|
31
|
+
"destination": self.destination,
|
|
32
|
+
"overwritten": self.overwritten,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _bundled_skill_roots() -> list[Path]:
|
|
37
|
+
repo_root = Path(__file__).resolve().parents[1]
|
|
38
|
+
return [
|
|
39
|
+
repo_root / ".agents" / "skills",
|
|
40
|
+
Path(sysconfig.get_path("data")) / ".agents" / "skills",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _find_source_skill(skill: str, source_root: Path | None) -> Path:
|
|
45
|
+
roots = [source_root] if source_root is not None else _bundled_skill_roots()
|
|
46
|
+
for root in roots:
|
|
47
|
+
candidate = root / skill
|
|
48
|
+
if (candidate / "SKILL.md").is_file():
|
|
49
|
+
return candidate
|
|
50
|
+
|
|
51
|
+
searched = ", ".join(str(root) for root in roots)
|
|
52
|
+
raise SkillInstallError(f"Skill {skill!r} was not found. Searched: {searched}")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _target_root(target: SkillTarget, project_dir: Path | None) -> Path:
|
|
56
|
+
if target == "user":
|
|
57
|
+
return Path.home() / ".agents" / "skills"
|
|
58
|
+
if target == "project":
|
|
59
|
+
return (project_dir or Path.cwd()) / ".agents" / "skills"
|
|
60
|
+
raise SkillInstallError("Target must be 'user' or 'project'")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def install_skill(
|
|
64
|
+
skill: str = "graph",
|
|
65
|
+
*,
|
|
66
|
+
target: SkillTarget = "user",
|
|
67
|
+
force: bool = False,
|
|
68
|
+
source_root: Path | None = None,
|
|
69
|
+
project_dir: Path | None = None,
|
|
70
|
+
) -> SkillInstallResult:
|
|
71
|
+
"""Copy a bundled skill into the selected user or project skill directory."""
|
|
72
|
+
source = _find_source_skill(skill, source_root)
|
|
73
|
+
destination = _target_root(target, project_dir) / skill
|
|
74
|
+
overwritten = destination.exists()
|
|
75
|
+
|
|
76
|
+
if overwritten and not force:
|
|
77
|
+
raise SkillInstallError(
|
|
78
|
+
f"Skill {skill!r} already exists at {destination}. Use --force to overwrite it."
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
if overwritten:
|
|
82
|
+
shutil.rmtree(destination)
|
|
83
|
+
|
|
84
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
shutil.copytree(source, destination)
|
|
86
|
+
|
|
87
|
+
return SkillInstallResult(
|
|
88
|
+
skill=skill,
|
|
89
|
+
target=target,
|
|
90
|
+
source=str(source),
|
|
91
|
+
destination=str(destination),
|
|
92
|
+
overwritten=overwritten,
|
|
93
|
+
)
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: graph
|
|
3
|
+
description: Use the AgentGraph CLI to query the local knowledge graph, inspect connectors, fetch entities, traverse relationships, and configure MCP access.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# /graph — AgentGraph CLI skill
|
|
7
|
+
|
|
8
|
+
Use the `agentgraph` CLI to query the local knowledge graph. Always prefer the CLI over direct Python/DB access.
|
|
9
|
+
|
|
10
|
+
## Commands
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
# Semantic search across entities
|
|
14
|
+
agentgraph search "<query>" [--type <type>] [--platform <platform>] [--limit N] [--json]
|
|
15
|
+
|
|
16
|
+
# Fetch full existing entity details by ID, UUID prefix, platform ref, or URL
|
|
17
|
+
agentgraph get <entity-id|platform/ref|url> --resolve [--json]
|
|
18
|
+
|
|
19
|
+
# List edges for an entity
|
|
20
|
+
agentgraph edges <entity-id|platform/ref> [--type <edge-type>] [--direction in|out|both] [--json]
|
|
21
|
+
|
|
22
|
+
# Traverse the graph from a starting entity
|
|
23
|
+
agentgraph traverse <entity-id|platform/ref> --resolve [--depth N] [--json]
|
|
24
|
+
|
|
25
|
+
# Filter entities by type and metadata
|
|
26
|
+
agentgraph query --type <entity-type> [--filter key=value] [--since 12h|30m|2d] [--mine] [--has-attachments] [--limit N] [--order-by created_at|updated_at|last_accessed] [--json]
|
|
27
|
+
|
|
28
|
+
# Trigger a connector fetch for a platform entity (by platform + platform-specific ID)
|
|
29
|
+
agentgraph fetch <platform> <resource-id> [--json]
|
|
30
|
+
|
|
31
|
+
# Trigger a connector re-fetch for an entity by its internal UUID
|
|
32
|
+
agentgraph fetch-entity <entity-id> [--json]
|
|
33
|
+
|
|
34
|
+
# Download an entity's source file using connector auth, including Gmail attachment Document stubs
|
|
35
|
+
agentgraph download <entity-id|platform/ref> [--output <file-or-dir>] [--json]
|
|
36
|
+
|
|
37
|
+
# Bookmark an entity or retrieve and bookmark an HTTP(S) URL; use --remove to clear bookmark protection
|
|
38
|
+
agentgraph bookmark <entity-id|platform/ref|url> [--remove] [--json]
|
|
39
|
+
|
|
40
|
+
# Delete an entity from the graph
|
|
41
|
+
agentgraph delete <entity-id|platform/ref|url> [--json]
|
|
42
|
+
|
|
43
|
+
# Merge duplicate Person entities that refer to the same human
|
|
44
|
+
agentgraph unify-persons <primary-person-id> <duplicate-person-id>... [--json]
|
|
45
|
+
|
|
46
|
+
# Queue a background poll for one or all connectors
|
|
47
|
+
agentgraph poll [<source>] [--json] # source: slack, gmail, discord, drive, rss — omit for all; reports already_running and skipped auth failures
|
|
48
|
+
|
|
49
|
+
# Run a one-shot bulk ingest for a connector (all data within the retention window, beyond what poll covers)
|
|
50
|
+
agentgraph ingest <source> [--json] # e.g. gmail, rss
|
|
51
|
+
|
|
52
|
+
# List installed connectors and their sync status; credential fields are null for connectors like RSS/web
|
|
53
|
+
agentgraph connectors [--verify] [--json] # auth_provider, auth_status/auth_detail when applicable, auth_verified, url_patterns, polls, poll_delegates, polled_by, sync, last_synced_at
|
|
54
|
+
|
|
55
|
+
# Run a connector-owned command
|
|
56
|
+
agentgraph connector <source> <command> [args...] [--json] # e.g. agentgraph connector rss add https://simonwillison.net/atom/everything/
|
|
57
|
+
agentgraph connector <source> --help
|
|
58
|
+
agentgraph connector rss add <feed-url> [feed-url...] [--json] # validates feeds, rejects non-feeds without saving, and queues an RSS poll
|
|
59
|
+
agentgraph connector rss remove <feed-url> [feed-url...] [--json] # removes exact configured feed URLs
|
|
60
|
+
agentgraph connector rss import-opml <file.opml> [--all | --select 1,3-5] [--json] # omit flags for checkbox selection
|
|
61
|
+
|
|
62
|
+
# Show credential-backed auth provider state (dedupes shared providers like Google); add --verify for live provider API checks
|
|
63
|
+
agentgraph auth [--verify] [--json] status # provider, connectors[], auth_status/auth_detail, auth_verified, accounts[] including auth_method
|
|
64
|
+
|
|
65
|
+
# Authenticate connectors/providers
|
|
66
|
+
agentgraph auth google [--add] [--account <account-id>] # uses AgentGraph's packaged OAuth client
|
|
67
|
+
agentgraph auth slack [--method oauth|browser] [--client-id <client-id>] [--add] [--account <account-id>] # --client-id implies OAuth; without one, OAuth shows admin/app setup guidance
|
|
68
|
+
agentgraph auth slack --method browser [--xoxc-token <token>] [--d-cookie <cookie>] # explicit browser-session fallback; credential flags imply browser
|
|
69
|
+
agentgraph auth discord [--add] [--account <account-id>] # Discord bot token
|
|
70
|
+
agentgraph auth remove <provider> [--account <account-id>] [--json] # remove stored credentials; does not delete graph data
|
|
71
|
+
agentgraph connector rss add <feed-url> # RSS/Atom feed URLs are connector configuration, not auth
|
|
72
|
+
|
|
73
|
+
# Server
|
|
74
|
+
agentgraph serve [--reload]
|
|
75
|
+
agentgraph mcp-serve
|
|
76
|
+
agentgraph mcp-config # stdio config for Claude Desktop/Claude Code; ChatGPT uses a tunneled HTTPS /mcp endpoint
|
|
77
|
+
|
|
78
|
+
# Install the bundled AgentGraph skill into ~/.agents/skills or ./.agents/skills
|
|
79
|
+
agentgraph install-skill [graph] [--target user|project] [--force] [--json]
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
`--depth 0` returns only the requested entity; depths 1 through 4 include that many relationship hops.
|
|
83
|
+
|
|
84
|
+
## MCP tool equivalents
|
|
85
|
+
|
|
86
|
+
When using AgentGraph through MCP instead of the CLI, use these equivalent tools:
|
|
87
|
+
|
|
88
|
+
```text
|
|
89
|
+
agentgraph connectors -> list_connectors_tool(verify)
|
|
90
|
+
agentgraph auth [--json] status -> list_auth_providers_tool(verify) # credential-backed providers only
|
|
91
|
+
agentgraph auth <provider> ... -> authenticate_provider_tool(provider, args, account_id, add)
|
|
92
|
+
agentgraph auth remove ... -> remove_auth_provider_tool(provider, account_id)
|
|
93
|
+
agentgraph connector <source> ... -> run_connector_command_tool(source, args)
|
|
94
|
+
agentgraph search ... -> search_entities_tool(...)
|
|
95
|
+
agentgraph get ... -> get_entity_tool(entity_id)
|
|
96
|
+
agentgraph edges ... -> get_edges_tool(entity_id, edge_type, direction)
|
|
97
|
+
agentgraph traverse ... -> traverse_graph_tool(entity_id, max_depth)
|
|
98
|
+
agentgraph query ... -> query_by_filter_tool(...)
|
|
99
|
+
agentgraph fetch ... -> fetch_entity_tool(platform, resource_id)
|
|
100
|
+
agentgraph fetch-entity ... -> fetch_entity_by_id_tool(entity_id)
|
|
101
|
+
agentgraph download ... -> download_entity_tool(entity_id, output_path)
|
|
102
|
+
agentgraph poll [source] -> poll_connectors_tool(source) # returns polled, already_running, and skipped lists
|
|
103
|
+
agentgraph ingest <source> -> ingest_connector_tool(source)
|
|
104
|
+
agentgraph bookmark ... -> bookmark_entity_tool(entity_id, bookmarked)
|
|
105
|
+
agentgraph delete ... -> delete_entity_tool(entity_id)
|
|
106
|
+
agentgraph unify-persons ... -> unify_persons_tool(primary_entity_id, duplicate_entity_ids)
|
|
107
|
+
agentgraph install-skill ... -> install_skill_tool(skill, target, force)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## Entity types
|
|
111
|
+
|
|
112
|
+
| Type | Contains |
|
|
113
|
+
|---|---|
|
|
114
|
+
| `Message` | Chat messages (Discord, Slack). **Chat images and file uploads are attachments on Message entities** — stored in `metadata.attachments` (JSON array with `url`, `filename`, `content_type`, `width`, `height`). Use `--has-attachments` to filter to messages with files. |
|
|
115
|
+
| `Document` | Text documents (Google Docs, etc.) and Gmail attachment stubs. Gmail attachment stubs are referenced by their owning `Thread` and can be downloaded with `agentgraph download`. |
|
|
116
|
+
| `Channel` | Chat channels and DM threads. |
|
|
117
|
+
| `Task` | Tasks or to-do items. |
|
|
118
|
+
| `Project` | Project/repository containers. |
|
|
119
|
+
|
|
120
|
+
To find chat images uploaded this week: `agentgraph query --type Message --has-attachments --since 7d --json`
|
|
121
|
+
|
|
122
|
+
To download a Gmail attachment: re-fetch the Gmail thread, traverse one hop to
|
|
123
|
+
find referenced Gmail `Document` stubs, then download the attachment document:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
agentgraph fetch-entity <gmail-thread-entity-id>
|
|
127
|
+
agentgraph traverse <gmail-thread-entity-id> --depth 1 --json
|
|
128
|
+
agentgraph download <attachment-document-entity-id> --output <file-or-dir>
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Notes
|
|
132
|
+
|
|
133
|
+
- Graph data commands require the AgentGraph server. If a command reports the server is unavailable, run `agentgraph serve`.
|
|
134
|
+
- Use `--json` when you need to parse results programmatically
|
|
135
|
+
- List-style commands (`search`, `query`, and graph viewer/browse results) return bounded content snippets with `content_truncated` when applicable. Use `agentgraph get <entity-id> --json` for full entity content.
|
|
136
|
+
- MCP `search_entities_tool` and `query_by_filter_tool` default to `refresh=false` so connector-owned network enrichment does not slow normal reads. Set `refresh=true` only when fresh connector-owned presentation metadata is needed.
|
|
137
|
+
- Bookmark targets accept: full UUID, UUID prefix, platform ref (`slack/T123/C123`, `gdocs/doc-id`, `discord/dm/456`), or HTTP(S) URL
|
|
138
|
+
- `agentgraph bookmark --remove <entity-id|platform/ref|url>` clears bookmark protection for existing graph entities
|
|
139
|
+
- Delete targets accept: full UUID, UUID prefix, platform ref, or HTTP(S) URL. Connected edges are removed with the entity.
|
|
140
|
+
- Use `agentgraph download` for source files stored behind connector auth, such as Drive PDFs, exported Google Docs/Sheets, or Gmail attachment `Document` stubs
|
|
141
|
+
- `agentgraph fetch` and `agentgraph fetch-entity` persist the connector's complete returned batch before reporting counts; content-rich resources such as RSS feeds may take several minutes
|
|
142
|
+
- Use `agentgraph bookmark` for entities or HTTP(S) URLs that should survive retention-window garbage collection
|
|
143
|
+
- Use `agentgraph unify-persons` only after confirming two or more `Person` entities are the same human; the first argument is the canonical person to keep. Without `--json`, the command displays the updated canonical Person, including merged identity metadata and duplicate identities.
|
|
144
|
+
- `polls: false` does not always mean stale: check `polled_by` / `sync` for connectors refreshed by another connector, e.g. `gdocs` and `gsheets` are refreshed via the `gdrive` Drive Changes poll
|
|
145
|
+
- Server logs go to stdout unless the process manager redirects them elsewhere
|
|
146
|
+
|
|
147
|
+
## Stub Entities
|
|
148
|
+
|
|
149
|
+
An entity is a **stub** when it has no title and no content — it was referenced in an edge but never fetched from its source. Using `--resolve` (default for `get` and `traverse`) automatically fetches stubs from their source before returning. If you omitted `--resolve` and get empty results, re-run with it or use `agentgraph fetch-entity <entity-id>` then re-fetch.
|
|
150
|
+
|
|
151
|
+
## Workflow
|
|
152
|
+
|
|
153
|
+
When the user asks about graph data:
|
|
154
|
+
1. Run `agentgraph connectors --json` to verify the relevant connector is installed and to inspect its last sync state
|
|
155
|
+
2. Run `agentgraph auth --json status` to inspect local provider-level authentication state, especially for shared auth like Google
|
|
156
|
+
3. If credential validity is uncertain, run `agentgraph auth --verify --json status` or `agentgraph connectors --verify --json` to live-check provider APIs for credential-backed connectors
|
|
157
|
+
4. If Google has `auth_status: "invalid"` or `"missing"`, tell the user to run `agentgraph auth google`
|
|
158
|
+
5. Run the appropriate `agentgraph` command with `--json` to get structured output
|
|
159
|
+
6. Use `edges` or `traverse` to follow relationships when needed
|