world-model-optimizer 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_waterfall/LICENSE +21 -0
- llm_waterfall/__init__.py +53 -0
- llm_waterfall/adapters/__init__.py +36 -0
- llm_waterfall/adapters/anthropic.py +105 -0
- llm_waterfall/adapters/aws_mantle.py +47 -0
- llm_waterfall/adapters/azure_openai.py +71 -0
- llm_waterfall/adapters/base.py +51 -0
- llm_waterfall/adapters/bedrock.py +309 -0
- llm_waterfall/adapters/openai.py +130 -0
- llm_waterfall/classify.py +184 -0
- llm_waterfall/pricing.py +110 -0
- llm_waterfall/py.typed +0 -0
- llm_waterfall/types.py +295 -0
- llm_waterfall/waterfall.py +255 -0
- wmo/__init__.py +38 -0
- wmo/agents/__init__.py +7 -0
- wmo/agents/default.py +29 -0
- wmo/agents/meta.py +55 -0
- wmo/agents/optimizer.py +55 -0
- wmo/agents/project.py +928 -0
- wmo/cli/__init__.py +5 -0
- wmo/cli/agent_session.py +1123 -0
- wmo/cli/app.py +2489 -0
- wmo/cli/e2b_cmds.py +212 -0
- wmo/cli/eval_closed_loop.py +207 -0
- wmo/cli/harness_app.py +1147 -0
- wmo/cli/harness_distill.py +659 -0
- wmo/cli/hosted_session.py +880 -0
- wmo/cli/ingest_cmd.py +165 -0
- wmo/cli/model_roles.py +82 -0
- wmo/cli/platform_cmds.py +372 -0
- wmo/cli/route_app.py +274 -0
- wmo/cli/session_state.py +243 -0
- wmo/cli/ui.py +1107 -0
- wmo/cli/workspace_sync.py +504 -0
- wmo/config/__init__.py +60 -0
- wmo/config/card.py +129 -0
- wmo/config/config.py +367 -0
- wmo/config/dotenv.py +67 -0
- wmo/config/settings.py +128 -0
- wmo/config/store.py +177 -0
- wmo/conftest.py +19 -0
- wmo/connect/__init__.py +88 -0
- wmo/connect/apps.py +78 -0
- wmo/connect/brave.py +284 -0
- wmo/connect/connector.py +79 -0
- wmo/connect/credentials.py +164 -0
- wmo/connect/github.py +321 -0
- wmo/connect/google.py +627 -0
- wmo/connect/notion.py +790 -0
- wmo/connect/oauth.py +461 -0
- wmo/connect/slack.py +555 -0
- wmo/connect/store.py +199 -0
- wmo/connect/types.py +156 -0
- wmo/core/__init__.py +21 -0
- wmo/core/parsing.py +281 -0
- wmo/core/render.py +271 -0
- wmo/core/text.py +40 -0
- wmo/core/types.py +116 -0
- wmo/distill/__init__.py +14 -0
- wmo/distill/agents.py +140 -0
- wmo/distill/config.py +1006 -0
- wmo/distill/cost.py +437 -0
- wmo/distill/data.py +921 -0
- wmo/distill/deadlines.py +254 -0
- wmo/distill/fake_tinker.py +734 -0
- wmo/distill/gate.py +122 -0
- wmo/distill/loop.py +3499 -0
- wmo/distill/renderers.py +399 -0
- wmo/distill/rendering.py +620 -0
- wmo/distill/rollouts.py +726 -0
- wmo/distill/samples.py +195 -0
- wmo/distill/store.py +829 -0
- wmo/distill/teacher.py +714 -0
- wmo/distill/tokens.py +535 -0
- wmo/distill/tracking.py +552 -0
- wmo/distill/tripwire.py +411 -0
- wmo/distill/xtoken/byte_offsets.py +152 -0
- wmo/distill/xtoken/chunks.py +457 -0
- wmo/distill/xtoken/prompt_logprobs.py +475 -0
- wmo/distill/xtoken/teacher_render.py +346 -0
- wmo/engine/__init__.py +28 -0
- wmo/engine/autoconfig.py +367 -0
- wmo/engine/build.py +346 -0
- wmo/engine/demo.py +77 -0
- wmo/engine/eval_suites.py +245 -0
- wmo/engine/grounding.py +491 -0
- wmo/engine/knowledge.py +291 -0
- wmo/engine/loader.py +36 -0
- wmo/engine/play.py +92 -0
- wmo/engine/prompts.py +99 -0
- wmo/engine/replay.py +443 -0
- wmo/engine/reporting.py +58 -0
- wmo/engine/workspace.py +468 -0
- wmo/engine/world_model.py +568 -0
- wmo/env/__init__.py +22 -0
- wmo/env/base.py +121 -0
- wmo/env/closed_loop.py +229 -0
- wmo/env/episode.py +107 -0
- wmo/env/llm_agent.py +93 -0
- wmo/env/scenarios.py +73 -0
- wmo/evals/__init__.py +52 -0
- wmo/evals/agreement.py +110 -0
- wmo/evals/base.py +45 -0
- wmo/evals/closed_loop.py +480 -0
- wmo/evals/failover.py +96 -0
- wmo/evals/gold.py +127 -0
- wmo/evals/grid.py +394 -0
- wmo/evals/grid_plot.py +205 -0
- wmo/evals/harbor/__init__.py +27 -0
- wmo/evals/harbor/agent.py +573 -0
- wmo/evals/harbor/ctrf.py +171 -0
- wmo/evals/harbor/e2b_environment.py +587 -0
- wmo/evals/harbor/e2b_template_policy.py +144 -0
- wmo/evals/harbor/scorer.py +875 -0
- wmo/evals/harbor/tasks.py +140 -0
- wmo/evals/open_loop.py +194 -0
- wmo/evals/tasks.py +53 -0
- wmo/harness/__init__.py +51 -0
- wmo/harness/code_runtime.py +288 -0
- wmo/harness/create.py +1191 -0
- wmo/harness/delta.py +220 -0
- wmo/harness/doc.py +556 -0
- wmo/harness/e2b_ledger.py +342 -0
- wmo/harness/e2b_reap.py +476 -0
- wmo/harness/e2b_sandbox.py +350 -0
- wmo/harness/environment.py +35 -0
- wmo/harness/live_session.py +543 -0
- wmo/harness/mutate.py +343 -0
- wmo/harness/pi_e2b.py +1710 -0
- wmo/harness/pi_entry/entry.ts +268 -0
- wmo/harness/pi_entry/runner_frames.ts +92 -0
- wmo/harness/pi_entry/runner_live.ts +587 -0
- wmo/harness/pi_entry/runner_service.ts +270 -0
- wmo/harness/pi_entry/runner_stdio.ts +374 -0
- wmo/harness/pi_entry/runner_termination.ts +142 -0
- wmo/harness/pi_local.py +262 -0
- wmo/harness/pi_runtime.py +495 -0
- wmo/harness/pi_vendor.py +65 -0
- wmo/harness/population.py +509 -0
- wmo/harness/project_proposer.py +569 -0
- wmo/harness/proposer.py +977 -0
- wmo/harness/runner_link.py +619 -0
- wmo/harness/runtime.py +389 -0
- wmo/harness/scoring.py +247 -0
- wmo/harness/skills.py +116 -0
- wmo/harness/source_tree.py +319 -0
- wmo/harness/store.py +176 -0
- wmo/harness/tools.py +105 -0
- wmo/harness/vendor/manifest.sha256 +58 -0
- wmo/harness/vendor/pi-agent/CHANGELOG.md +556 -0
- wmo/harness/vendor/pi-agent/LICENSE +21 -0
- wmo/harness/vendor/pi-agent/README.md +488 -0
- wmo/harness/vendor/pi-agent/VENDOR.md +39 -0
- wmo/harness/vendor/pi-agent/docs/agent-harness.md +486 -0
- wmo/harness/vendor/pi-agent/docs/durable-harness.md +212 -0
- wmo/harness/vendor/pi-agent/docs/hooks.md +445 -0
- wmo/harness/vendor/pi-agent/docs/models.md +966 -0
- wmo/harness/vendor/pi-agent/docs/observability.md +376 -0
- wmo/harness/vendor/pi-agent/package.json +60 -0
- wmo/harness/vendor/pi-agent/src/agent-loop.ts +748 -0
- wmo/harness/vendor/pi-agent/src/agent.ts +575 -0
- wmo/harness/vendor/pi-agent/src/harness/agent-harness.ts +1029 -0
- wmo/harness/vendor/pi-agent/src/harness/compaction/branch-summarization.ts +261 -0
- wmo/harness/vendor/pi-agent/src/harness/compaction/compaction.ts +747 -0
- wmo/harness/vendor/pi-agent/src/harness/compaction/utils.ts +144 -0
- wmo/harness/vendor/pi-agent/src/harness/env/nodejs.ts +550 -0
- wmo/harness/vendor/pi-agent/src/harness/messages.ts +164 -0
- wmo/harness/vendor/pi-agent/src/harness/prompt-templates.ts +267 -0
- wmo/harness/vendor/pi-agent/src/harness/session/jsonl-repo.ts +177 -0
- wmo/harness/vendor/pi-agent/src/harness/session/jsonl-storage.ts +293 -0
- wmo/harness/vendor/pi-agent/src/harness/session/memory-repo.ts +50 -0
- wmo/harness/vendor/pi-agent/src/harness/session/memory-storage.ts +131 -0
- wmo/harness/vendor/pi-agent/src/harness/session/repo-utils.ts +51 -0
- wmo/harness/vendor/pi-agent/src/harness/session/session.ts +267 -0
- wmo/harness/vendor/pi-agent/src/harness/session/uuid.ts +54 -0
- wmo/harness/vendor/pi-agent/src/harness/skills.ts +375 -0
- wmo/harness/vendor/pi-agent/src/harness/system-prompt.ts +34 -0
- wmo/harness/vendor/pi-agent/src/harness/types.ts +836 -0
- wmo/harness/vendor/pi-agent/src/harness/utils/shell-output.ts +135 -0
- wmo/harness/vendor/pi-agent/src/harness/utils/truncate.ts +344 -0
- wmo/harness/vendor/pi-agent/src/index.ts +44 -0
- wmo/harness/vendor/pi-agent/src/node.ts +2 -0
- wmo/harness/vendor/pi-agent/src/proxy.ts +367 -0
- wmo/harness/vendor/pi-agent/src/types.ts +428 -0
- wmo/harness/vendor/pi-agent/test/agent-loop.test.ts +1351 -0
- wmo/harness/vendor/pi-agent/test/agent.test.ts +699 -0
- wmo/harness/vendor/pi-agent/test/e2e.test.ts +404 -0
- wmo/harness/vendor/pi-agent/test/harness/agent-harness-stream.test.ts +213 -0
- wmo/harness/vendor/pi-agent/test/harness/agent-harness.test.ts +608 -0
- wmo/harness/vendor/pi-agent/test/harness/compaction.test.ts +655 -0
- wmo/harness/vendor/pi-agent/test/harness/nodejs-env.test.ts +321 -0
- wmo/harness/vendor/pi-agent/test/harness/prompt-templates.test.ts +90 -0
- wmo/harness/vendor/pi-agent/test/harness/repo.test.ts +68 -0
- wmo/harness/vendor/pi-agent/test/harness/resource-formatting.test.ts +24 -0
- wmo/harness/vendor/pi-agent/test/harness/session-test-utils.ts +55 -0
- wmo/harness/vendor/pi-agent/test/harness/session-uuid.test.ts +50 -0
- wmo/harness/vendor/pi-agent/test/harness/session.test.ts +156 -0
- wmo/harness/vendor/pi-agent/test/harness/skills.test.ts +116 -0
- wmo/harness/vendor/pi-agent/test/harness/storage.test.ts +299 -0
- wmo/harness/vendor/pi-agent/test/harness/system-prompt.test.ts +66 -0
- wmo/harness/vendor/pi-agent/test/harness/truncate.test.ts +169 -0
- wmo/harness/vendor/pi-agent/test/scratch/simple.ts +72 -0
- wmo/harness/vendor/pi-agent/test/utils/calculate.ts +32 -0
- wmo/harness/vendor/pi-agent/test/utils/get-current-time.ts +46 -0
- wmo/harness/vendor/pi-agent/tsconfig.build.json +13 -0
- wmo/harness/vendor/pi-agent/vitest.config.ts +19 -0
- wmo/harness/vendor/pi-agent/vitest.harness.config.ts +28 -0
- wmo/harness/vendor/vendor_pi.sh +59 -0
- wmo/harness/workspace_patch.py +270 -0
- wmo/ingest/__init__.py +47 -0
- wmo/ingest/adapter.py +72 -0
- wmo/ingest/base.py +114 -0
- wmo/ingest/braintrust.py +339 -0
- wmo/ingest/detect.py +126 -0
- wmo/ingest/langfuse.py +291 -0
- wmo/ingest/langsmith.py +444 -0
- wmo/ingest/mastra.py +330 -0
- wmo/ingest/messages.py +170 -0
- wmo/ingest/normalize.py +679 -0
- wmo/ingest/otel_genai.py +69 -0
- wmo/ingest/otel_writer.py +100 -0
- wmo/ingest/phoenix.py +150 -0
- wmo/ingest/postgres.py +246 -0
- wmo/ingest/posthog.py +320 -0
- wmo/ingest/quality.py +28 -0
- wmo/ingest/stream.py +209 -0
- wmo/ingest/testdata/sample_otlp.json +60 -0
- wmo/ingest/testdata/sample_spans.jsonl +3 -0
- wmo/optimize/__init__.py +25 -0
- wmo/optimize/base.py +143 -0
- wmo/optimize/gepa.py +806 -0
- wmo/optimize/judge.py +262 -0
- wmo/optimize/judge_quality.py +359 -0
- wmo/optimize/knn.py +468 -0
- wmo/optimize/numeric.py +152 -0
- wmo/optimize/outcomes.py +103 -0
- wmo/optimize/policy.py +669 -0
- wmo/optimize/report.py +231 -0
- wmo/optimize/reward.py +129 -0
- wmo/optimize/routing.py +373 -0
- wmo/platform/__init__.py +6 -0
- wmo/platform/auth.py +115 -0
- wmo/platform/client.py +551 -0
- wmo/platform/credentials.py +126 -0
- wmo/platform/transfer.py +158 -0
- wmo/providers/__init__.py +40 -0
- wmo/providers/_bedrock_chat.py +155 -0
- wmo/providers/_openai_common.py +182 -0
- wmo/providers/_responses_common.py +472 -0
- wmo/providers/anthropic.py +134 -0
- wmo/providers/azure_openai.py +296 -0
- wmo/providers/base.py +300 -0
- wmo/providers/bedrock.py +312 -0
- wmo/providers/models.py +205 -0
- wmo/providers/openai.py +143 -0
- wmo/providers/openai_responses.py +240 -0
- wmo/providers/pool.py +170 -0
- wmo/providers/registry.py +73 -0
- wmo/providers/retry.py +151 -0
- wmo/providers/tinker.py +936 -0
- wmo/providers/waterfall.py +336 -0
- wmo/research/__init__.py +81 -0
- wmo/research/ablation.py +133 -0
- wmo/research/concurrency_plot.py +523 -0
- wmo/research/concurrency_run.py +240 -0
- wmo/research/concurrency_scaling.py +270 -0
- wmo/research/gepa_scaling.py +274 -0
- wmo/research/pipeline.py +198 -0
- wmo/research/scaling_split.py +82 -0
- wmo/research/scenario_fidelity.py +198 -0
- wmo/research/scenario_recovery.py +92 -0
- wmo/research/seed_stability.py +90 -0
- wmo/research/trace_scaling.py +348 -0
- wmo/retrieval/__init__.py +6 -0
- wmo/retrieval/embedders.py +105 -0
- wmo/retrieval/leakfree.py +52 -0
- wmo/retrieval/retriever.py +173 -0
- wmo/scenarios/__init__.py +58 -0
- wmo/scenarios/builder.py +152 -0
- wmo/scenarios/mining/__init__.py +27 -0
- wmo/scenarios/mining/clustering.py +171 -0
- wmo/scenarios/mining/facets.py +226 -0
- wmo/scenarios/mining/selection.py +220 -0
- wmo/scenarios/synthesis/__init__.py +6 -0
- wmo/scenarios/synthesis/scenario_set.py +63 -0
- wmo/scenarios/synthesis/synthesizer.py +85 -0
- wmo/scenarios/verification/__init__.py +17 -0
- wmo/scenarios/verification/judge.py +97 -0
- wmo/scenarios/verification/verify.py +135 -0
- wmo/serving/__init__.py +5 -0
- wmo/serving/builds.py +451 -0
- wmo/serving/chat.py +878 -0
- wmo/serving/endpoint_config.py +64 -0
- wmo/serving/savings.py +250 -0
- wmo/serving/server.py +553 -0
- wmo/serving/traces_source.py +206 -0
- wmo/telemetry.py +213 -0
- wmo/tracking/__init__.py +36 -0
- wmo/tracking/clock.py +24 -0
- wmo/tracking/metered.py +125 -0
- wmo/tracking/pricing.py +99 -0
- wmo/tracking/store.py +31 -0
- wmo/tracking/tracker.py +149 -0
- world_model_optimizer-0.2.0.dist-info/METADATA +203 -0
- world_model_optimizer-0.2.0.dist-info/RECORD +308 -0
- world_model_optimizer-0.2.0.dist-info/WHEEL +4 -0
- world_model_optimizer-0.2.0.dist-info/entry_points.txt +2 -0
wmo/connect/notion.py
ADDED
|
@@ -0,0 +1,790 @@
|
|
|
1
|
+
"""Notion context connector: the official remote MCP server by default, REST as fallback.
|
|
2
|
+
|
|
3
|
+
Two credential paths:
|
|
4
|
+
|
|
5
|
+
* Default (zero registration): OAuth against the hosted Notion MCP server
|
|
6
|
+
(https://mcp.notion.com/mcp) via the MCP SDK's ``OAuthClientProvider`` with dynamic client
|
|
7
|
+
registration and a localhost redirect. Tokens persist as a normal oauth-kind ``ConnectorAuth``
|
|
8
|
+
and the registered-client record rides along in ``extra["mcp_client_info"]``, so refreshes
|
|
9
|
+
survive across runs. Pulls call the server's search/fetch tools.
|
|
10
|
+
* Fallback: a pasted internal-integration secret (``ntn_``/``secret_`` prefixed) or the
|
|
11
|
+
``$WMO_NOTION_TOKEN`` env var (token-kind auth) talks to the REST API at api.notion.com.
|
|
12
|
+
|
|
13
|
+
The mcp SDK is the optional ``connectors`` extra: this module imports and registers without it,
|
|
14
|
+
and only the MCP code paths import it (lazily, mirroring the e2b guard in
|
|
15
|
+
``wmo/harness/e2b_sandbox.py``). Asyncio stays contained here: the public connector surface is
|
|
16
|
+
synchronous.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
import json
|
|
23
|
+
import logging
|
|
24
|
+
import os
|
|
25
|
+
import threading
|
|
26
|
+
import time
|
|
27
|
+
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
|
|
28
|
+
from contextlib import asynccontextmanager
|
|
29
|
+
from dataclasses import dataclass
|
|
30
|
+
from datetime import UTC, datetime, timedelta
|
|
31
|
+
from typing import TYPE_CHECKING, cast
|
|
32
|
+
|
|
33
|
+
import httpx
|
|
34
|
+
from pydantic import ValidationError
|
|
35
|
+
|
|
36
|
+
from wmo.connect.connector import ConnectUI, register_connector
|
|
37
|
+
from wmo.connect.credentials import list_connected, save_connector_auth, token_env_var
|
|
38
|
+
from wmo.connect.oauth import LoopbackServer, serve_until
|
|
39
|
+
from wmo.connect.types import (
|
|
40
|
+
ConnectError,
|
|
41
|
+
ConnectorAuth,
|
|
42
|
+
ContextItem,
|
|
43
|
+
ItemKind,
|
|
44
|
+
PullQuery,
|
|
45
|
+
opt_str,
|
|
46
|
+
transport_errors,
|
|
47
|
+
)
|
|
48
|
+
from wmo.core.types import JsonObject, JsonValue
|
|
49
|
+
|
|
50
|
+
if TYPE_CHECKING:
|
|
51
|
+
from mcp.client.session import ClientSession
|
|
52
|
+
from mcp.shared.auth import OAuthClientInformationFull, OAuthToken
|
|
53
|
+
from mcp.types import CallToolResult
|
|
54
|
+
|
|
55
|
+
logger = logging.getLogger(__name__)
|
|
56
|
+
|
|
57
|
+
NOTION_MCP_URL = "https://mcp.notion.com/mcp"
|
|
58
|
+
|
|
59
|
+
_NAME = "notion"
|
|
60
|
+
_API_BASE = "https://api.notion.com"
|
|
61
|
+
_API_HOST = "api.notion.com"
|
|
62
|
+
_NOTION_VERSION = "2022-06-28"
|
|
63
|
+
_CLIENT_INFO_KEY = "mcp_client_info"
|
|
64
|
+
_PAGE_SIZE = 100 # Notion's maximum page_size, for both search and block children
|
|
65
|
+
_MAX_BLOCK_DEPTH = 1 # page children, plus one level into blocks that have children
|
|
66
|
+
_REST_TIMEOUT_SECONDS = 30.0
|
|
67
|
+
_OAUTH_TIMEOUT_SECONDS = 300.0
|
|
68
|
+
_BODY_KEYS = ("text", "content", "body", "markdown")
|
|
69
|
+
_MCP_INSTALL_HINT = (
|
|
70
|
+
"the mcp SDK is not installed; run `uv sync --extra connectors` (or "
|
|
71
|
+
"pip install 'world-model-optimizer[connectors]') to use the Notion MCP connector"
|
|
72
|
+
)
|
|
73
|
+
_RECONNECT_HINT = "the connection must be reauthorized"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# -- small shared helpers -------------------------------------------------------------------
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _parse_iso(value: str | None) -> datetime | None:
|
|
80
|
+
"""Parse an ISO-8601 date or datetime; naive values count as UTC, garbage becomes None."""
|
|
81
|
+
if not value:
|
|
82
|
+
return None
|
|
83
|
+
try:
|
|
84
|
+
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
85
|
+
except ValueError:
|
|
86
|
+
return None
|
|
87
|
+
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _first_str(source: JsonObject, keys: tuple[str, ...]) -> str | None:
|
|
91
|
+
"""The first non-empty string value among `keys` in `source`."""
|
|
92
|
+
for key in keys:
|
|
93
|
+
value = source.get(key)
|
|
94
|
+
if isinstance(value, str) and value:
|
|
95
|
+
return value
|
|
96
|
+
return None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _stored_oauth_auth() -> ConnectorAuth | None:
|
|
100
|
+
"""The file-stored notion credential ($WMO_NOTION_TOKEN deliberately bypassed).
|
|
101
|
+
|
|
102
|
+
MCP tokens always live in the credential file; reading through `load_connector_auth` would
|
|
103
|
+
let an ambient env token shadow them in the middle of an OAuth flow.
|
|
104
|
+
"""
|
|
105
|
+
return list_connected().get(_NAME)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _remaining_seconds(expires_at: str | None) -> int | None:
|
|
109
|
+
"""Seconds until `expires_at` (floored at 0), or None when absent/unparseable."""
|
|
110
|
+
expires = _parse_iso(expires_at)
|
|
111
|
+
if expires is None:
|
|
112
|
+
return None
|
|
113
|
+
return max(0, int((expires - datetime.now(UTC)).total_seconds()))
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _absolute_expiry(expires_in: int | None) -> str | None:
|
|
117
|
+
"""A relative `expires_in` turned into an absolute ISO-8601 timestamp."""
|
|
118
|
+
if expires_in is None:
|
|
119
|
+
return None
|
|
120
|
+
moment = datetime.now(UTC) + timedelta(seconds=float(expires_in))
|
|
121
|
+
return moment.isoformat(timespec="seconds")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# -- MCP token storage ----------------------------------------------------------------------
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class _McpTokenStorage:
|
|
128
|
+
"""The MCP SDK's TokenStorage protocol backed by the wmo connector credential store.
|
|
129
|
+
|
|
130
|
+
`mcp.client.auth.TokenStorage` is a structural protocol, so this class defines the four
|
|
131
|
+
async methods without importing the SDK at class-definition time. OAuth tokens map onto
|
|
132
|
+
the standard `ConnectorAuth` fields under the "notion" table; the dynamic-client-
|
|
133
|
+
registration record rides along in `extra["mcp_client_info"]`.
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
async def get_tokens(self) -> OAuthToken | None:
|
|
137
|
+
"""The stored token in SDK shape, or None before the first authorization."""
|
|
138
|
+
try:
|
|
139
|
+
from mcp.shared.auth import OAuthToken
|
|
140
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
141
|
+
raise ImportError(_MCP_INSTALL_HINT) from exc
|
|
142
|
+
auth = _stored_oauth_auth()
|
|
143
|
+
if auth is None or auth.kind != "oauth" or not auth.access_token:
|
|
144
|
+
return None
|
|
145
|
+
return OAuthToken(
|
|
146
|
+
access_token=auth.access_token,
|
|
147
|
+
refresh_token=auth.refresh_token,
|
|
148
|
+
scope=" ".join(auth.scopes) or None,
|
|
149
|
+
expires_in=_remaining_seconds(auth.expires_at),
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
async def set_tokens(self, tokens: OAuthToken) -> None:
|
|
153
|
+
"""Persist fresh tokens, carrying the account and registration record forward."""
|
|
154
|
+
current = _stored_oauth_auth()
|
|
155
|
+
scopes = tokens.scope.split() if tokens.scope else (current.scopes if current else [])
|
|
156
|
+
save_connector_auth(
|
|
157
|
+
_NAME,
|
|
158
|
+
ConnectorAuth(
|
|
159
|
+
kind="oauth",
|
|
160
|
+
access_token=tokens.access_token,
|
|
161
|
+
refresh_token=tokens.refresh_token,
|
|
162
|
+
expires_at=_absolute_expiry(tokens.expires_in),
|
|
163
|
+
scopes=scopes,
|
|
164
|
+
account=current.account if current else None,
|
|
165
|
+
extra=current.extra if current else {},
|
|
166
|
+
),
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
async def get_client_info(self) -> OAuthClientInformationFull | None:
|
|
170
|
+
"""The persisted registration record, or None (a corrupt one triggers re-registration)."""
|
|
171
|
+
try:
|
|
172
|
+
from mcp.shared.auth import OAuthClientInformationFull
|
|
173
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
174
|
+
raise ImportError(_MCP_INSTALL_HINT) from exc
|
|
175
|
+
auth = _stored_oauth_auth()
|
|
176
|
+
raw = auth.extra.get(_CLIENT_INFO_KEY) if auth else None
|
|
177
|
+
if not isinstance(raw, dict):
|
|
178
|
+
return None
|
|
179
|
+
try:
|
|
180
|
+
return OAuthClientInformationFull.model_validate(raw)
|
|
181
|
+
except ValidationError:
|
|
182
|
+
logger.warning("discarding an unparseable %s record; re-registering", _CLIENT_INFO_KEY)
|
|
183
|
+
return None
|
|
184
|
+
|
|
185
|
+
async def set_client_info(self, client_info: OAuthClientInformationFull) -> None:
|
|
186
|
+
"""Persist the registration record (registration happens before the first tokens)."""
|
|
187
|
+
current = _stored_oauth_auth() or ConnectorAuth(kind="oauth", access_token="")
|
|
188
|
+
extra = dict(current.extra)
|
|
189
|
+
extra[_CLIENT_INFO_KEY] = client_info.model_dump(mode="json", exclude_none=True)
|
|
190
|
+
save_connector_auth(_NAME, current.model_copy(update={"extra": extra}))
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
# -- MCP session plumbing -------------------------------------------------------------------
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _mcp_failure(exc: BaseException) -> str:
|
|
197
|
+
"""One actionable message shape for every MCP-session failure."""
|
|
198
|
+
return (
|
|
199
|
+
f"the Notion MCP session failed: {exc}; {_RECONNECT_HINT} (or set "
|
|
200
|
+
f"${token_env_var(_NAME)} to use the REST API instead)"
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _leaf_exception(group: BaseExceptionGroup[BaseException]) -> BaseException:
|
|
205
|
+
"""The first non-group leaf of a (possibly nested) exception group."""
|
|
206
|
+
exc: BaseException = group
|
|
207
|
+
while isinstance(exc, BaseExceptionGroup):
|
|
208
|
+
exc = exc.exceptions[0]
|
|
209
|
+
return exc
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@asynccontextmanager
|
|
213
|
+
async def _mcp_session(
|
|
214
|
+
*,
|
|
215
|
+
redirect_uri: str | None = None,
|
|
216
|
+
redirect_handler: Callable[[str], Awaitable[None]] | None = None,
|
|
217
|
+
callback_handler: Callable[[], Awaitable[tuple[str, str | None]]] | None = None,
|
|
218
|
+
timeout: float = _OAUTH_TIMEOUT_SECONDS,
|
|
219
|
+
) -> AsyncIterator[ClientSession]:
|
|
220
|
+
"""An initialized MCP session against the hosted Notion server.
|
|
221
|
+
|
|
222
|
+
Without redirect/callback handlers the provider can still refresh a stored token, but a
|
|
223
|
+
full re-authorization fails, mapped to a ConnectError telling the user to reconnect.
|
|
224
|
+
"""
|
|
225
|
+
try:
|
|
226
|
+
from mcp.client.auth import (
|
|
227
|
+
OAuthClientProvider,
|
|
228
|
+
OAuthFlowError,
|
|
229
|
+
OAuthRegistrationError,
|
|
230
|
+
OAuthTokenError,
|
|
231
|
+
)
|
|
232
|
+
from mcp.client.session import ClientSession
|
|
233
|
+
from mcp.client.streamable_http import streamable_http_client
|
|
234
|
+
from mcp.shared.auth import OAuthClientMetadata
|
|
235
|
+
from mcp.shared.exceptions import McpError
|
|
236
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
237
|
+
raise ImportError(_MCP_INSTALL_HINT) from exc
|
|
238
|
+
|
|
239
|
+
metadata = OAuthClientMetadata.model_validate(
|
|
240
|
+
{
|
|
241
|
+
"client_name": "world-model-optimizer",
|
|
242
|
+
"redirect_uris": [redirect_uri or "http://127.0.0.1/callback"],
|
|
243
|
+
"grant_types": ["authorization_code", "refresh_token"],
|
|
244
|
+
"response_types": ["code"],
|
|
245
|
+
"token_endpoint_auth_method": "none",
|
|
246
|
+
}
|
|
247
|
+
)
|
|
248
|
+
provider = OAuthClientProvider(
|
|
249
|
+
server_url=NOTION_MCP_URL,
|
|
250
|
+
client_metadata=metadata,
|
|
251
|
+
storage=_McpTokenStorage(),
|
|
252
|
+
redirect_handler=redirect_handler,
|
|
253
|
+
callback_handler=callback_handler,
|
|
254
|
+
timeout=timeout,
|
|
255
|
+
)
|
|
256
|
+
try:
|
|
257
|
+
async with (
|
|
258
|
+
httpx.AsyncClient(
|
|
259
|
+
auth=provider,
|
|
260
|
+
follow_redirects=True,
|
|
261
|
+
timeout=httpx.Timeout(_REST_TIMEOUT_SECONDS, read=timeout),
|
|
262
|
+
) as http_client,
|
|
263
|
+
streamable_http_client(NOTION_MCP_URL, http_client=http_client) as streams,
|
|
264
|
+
):
|
|
265
|
+
read_stream, write_stream, _get_session_id = streams
|
|
266
|
+
async with ClientSession(read_stream, write_stream) as session:
|
|
267
|
+
await session.initialize()
|
|
268
|
+
yield session
|
|
269
|
+
except ConnectError:
|
|
270
|
+
raise
|
|
271
|
+
except BaseExceptionGroup as group:
|
|
272
|
+
leaf = _leaf_exception(group)
|
|
273
|
+
if isinstance(leaf, ConnectError):
|
|
274
|
+
raise leaf from group
|
|
275
|
+
raise ConnectError(_mcp_failure(leaf)) from group
|
|
276
|
+
except (OAuthFlowError, OAuthRegistrationError, OAuthTokenError, McpError) as exc:
|
|
277
|
+
raise ConnectError(_mcp_failure(exc)) from exc
|
|
278
|
+
except httpx.HTTPError as exc:
|
|
279
|
+
raise ConnectError(_mcp_failure(exc)) from exc
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
async def _authorize_and_identify(
|
|
283
|
+
ui: ConnectUI, server: LoopbackServer, redirect_uri: str, timeout: float
|
|
284
|
+
) -> str:
|
|
285
|
+
"""Drive the interactive MCP OAuth flow; returns the verify identity string."""
|
|
286
|
+
|
|
287
|
+
async def redirect_handler(url: str) -> None:
|
|
288
|
+
ui.open_url(url)
|
|
289
|
+
|
|
290
|
+
async def callback_handler() -> tuple[str, str | None]:
|
|
291
|
+
received = await asyncio.to_thread(server.received.wait, timeout)
|
|
292
|
+
if not received:
|
|
293
|
+
raise ConnectError(
|
|
294
|
+
f"timed out after {timeout:g}s waiting for the Notion OAuth callback; "
|
|
295
|
+
"re-run the command and approve access in the browser"
|
|
296
|
+
)
|
|
297
|
+
params = server.callback_params or {}
|
|
298
|
+
if "error" in params:
|
|
299
|
+
description = params.get("error_description")
|
|
300
|
+
detail = f"{params['error']}: {description}" if description else params["error"]
|
|
301
|
+
raise ConnectError(
|
|
302
|
+
f"notion authorization failed: {detail}; re-run the command and approve access"
|
|
303
|
+
)
|
|
304
|
+
code = params.get("code")
|
|
305
|
+
if not code:
|
|
306
|
+
raise ConnectError(
|
|
307
|
+
"the Notion OAuth callback carried no authorization code; re-run the command"
|
|
308
|
+
)
|
|
309
|
+
return code, params.get("state")
|
|
310
|
+
|
|
311
|
+
async with _mcp_session(
|
|
312
|
+
redirect_uri=redirect_uri,
|
|
313
|
+
redirect_handler=redirect_handler,
|
|
314
|
+
callback_handler=callback_handler,
|
|
315
|
+
timeout=timeout,
|
|
316
|
+
) as session:
|
|
317
|
+
return await _verify_via_session(session)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
# -- MCP tool discovery + result parsing ----------------------------------------------------
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _match_tool(names: Sequence[str], needle: str) -> str | None:
|
|
324
|
+
"""The best tool name for `needle`: an exact match first, else the first containing it."""
|
|
325
|
+
for name in names:
|
|
326
|
+
if name.lower() == needle:
|
|
327
|
+
return name
|
|
328
|
+
for name in names:
|
|
329
|
+
if needle in name.lower():
|
|
330
|
+
return name
|
|
331
|
+
return None
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
async def _verify_via_session(session: ClientSession) -> str:
|
|
335
|
+
"""The verify identity string: server reachability plus the advertised tool count."""
|
|
336
|
+
listing = await session.list_tools()
|
|
337
|
+
return f"Notion MCP ({len(listing.tools)} tools)"
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
async def _pull_via_session(session: ClientSession, query: PullQuery) -> list[ContextItem]:
|
|
341
|
+
"""Search-then-fetch against an initialized MCP session, capped at `query.limit`."""
|
|
342
|
+
if query.limit <= 0:
|
|
343
|
+
return []
|
|
344
|
+
listing = await session.list_tools()
|
|
345
|
+
names = [tool.name for tool in listing.tools]
|
|
346
|
+
search_tool = _match_tool(names, "search")
|
|
347
|
+
if search_tool is None:
|
|
348
|
+
available = ", ".join(sorted(names)) or "none"
|
|
349
|
+
raise ConnectError(
|
|
350
|
+
f"the Notion MCP server exposes no search tool (available: {available}); "
|
|
351
|
+
f"update wmo, or set ${token_env_var(_NAME)} to pull via the REST API instead"
|
|
352
|
+
)
|
|
353
|
+
result = await session.call_tool(search_tool, {"query": query.query or ""})
|
|
354
|
+
if result.isError:
|
|
355
|
+
raise ConnectError(
|
|
356
|
+
f"the Notion MCP tool {search_tool!r} failed: {_result_text(result)[:200]}; "
|
|
357
|
+
f"{_RECONNECT_HINT} if the credential is stale"
|
|
358
|
+
)
|
|
359
|
+
hits = _search_hits(result)[: query.limit]
|
|
360
|
+
fetch_tool = _match_tool(names, "fetch")
|
|
361
|
+
since = _parse_iso(query.since)
|
|
362
|
+
until = _parse_iso(query.until)
|
|
363
|
+
items: list[ContextItem] = []
|
|
364
|
+
for hit in hits:
|
|
365
|
+
item = await _fetch_item(session, fetch_tool, hit)
|
|
366
|
+
if _within_window(item, since, until):
|
|
367
|
+
items.append(item)
|
|
368
|
+
return items
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
async def _fetch_item(
|
|
372
|
+
session: ClientSession, fetch_tool: str | None, hit: _SearchHit
|
|
373
|
+
) -> ContextItem:
|
|
374
|
+
"""The full item for one search hit; degrades to hit-only data when fetch is unavailable."""
|
|
375
|
+
if fetch_tool is None or hit.id is None:
|
|
376
|
+
return _item_from_payloads([], hit)
|
|
377
|
+
result = await session.call_tool(fetch_tool, {"id": hit.id})
|
|
378
|
+
if result.isError:
|
|
379
|
+
logger.warning("notion MCP fetch of %s failed: %s", hit.id, _result_text(result)[:200])
|
|
380
|
+
return _item_from_payloads([], hit)
|
|
381
|
+
return _item_from_payloads(_tool_payloads(result), hit)
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
@dataclass
|
|
385
|
+
class _SearchHit:
|
|
386
|
+
"""One search-tool result: enough identity to fetch and to build a fallback item."""
|
|
387
|
+
|
|
388
|
+
id: str | None
|
|
389
|
+
title: str
|
|
390
|
+
url: str | None
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _result_text(result: CallToolResult) -> str:
|
|
394
|
+
"""The concatenated text blocks of a tool result (error details, plain-text fallbacks)."""
|
|
395
|
+
texts: list[str] = []
|
|
396
|
+
for block in result.content:
|
|
397
|
+
text = getattr(block, "text", None)
|
|
398
|
+
if isinstance(text, str):
|
|
399
|
+
texts.append(text)
|
|
400
|
+
return "\n".join(texts)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _tool_payloads(result: CallToolResult) -> list[JsonValue]:
|
|
404
|
+
"""Tool-result payloads: text blocks (JSON when a block parses), then structured content."""
|
|
405
|
+
payloads: list[JsonValue] = []
|
|
406
|
+
for block in result.content:
|
|
407
|
+
text = getattr(block, "text", None)
|
|
408
|
+
if not isinstance(text, str):
|
|
409
|
+
continue
|
|
410
|
+
try:
|
|
411
|
+
payloads.append(cast(JsonValue, json.loads(text)))
|
|
412
|
+
except ValueError:
|
|
413
|
+
payloads.append(text)
|
|
414
|
+
if result.structuredContent is not None:
|
|
415
|
+
payloads.append(cast(JsonValue, result.structuredContent))
|
|
416
|
+
return payloads
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _search_hits(result: CallToolResult) -> list[_SearchHit]:
|
|
420
|
+
"""Search-tool results normalized to (id, title, url) hits."""
|
|
421
|
+
return [
|
|
422
|
+
_hit_from_entry(entry)
|
|
423
|
+
for payload in _tool_payloads(result)
|
|
424
|
+
for entry in _entry_dicts(payload)
|
|
425
|
+
]
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _entry_dicts(payload: JsonValue) -> list[JsonObject]:
|
|
429
|
+
"""Result entries inside one payload: a {"results": [...]} wrapper, a list, or one entry."""
|
|
430
|
+
if isinstance(payload, dict):
|
|
431
|
+
results = payload.get("results")
|
|
432
|
+
if isinstance(results, list):
|
|
433
|
+
return [entry for entry in results if isinstance(entry, dict)]
|
|
434
|
+
if "id" in payload or "url" in payload:
|
|
435
|
+
return [payload]
|
|
436
|
+
return []
|
|
437
|
+
if isinstance(payload, list):
|
|
438
|
+
return [entry for entry in payload if isinstance(entry, dict)]
|
|
439
|
+
return []
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _hit_from_entry(entry: JsonObject) -> _SearchHit:
|
|
443
|
+
"""One search hit from one result entry (missing titles become "Untitled")."""
|
|
444
|
+
return _SearchHit(
|
|
445
|
+
id=opt_str(entry.get("id")),
|
|
446
|
+
title=opt_str(entry.get("title")) or "Untitled",
|
|
447
|
+
url=opt_str(entry.get("url")),
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _item_from_payloads(payloads: list[JsonValue], hit: _SearchHit) -> ContextItem:
|
|
452
|
+
"""The normalized PAGE item from fetch-tool payloads (or from the search hit alone)."""
|
|
453
|
+
doc = next((payload for payload in payloads if isinstance(payload, dict)), None)
|
|
454
|
+
fallback_body = "\n\n".join(p for p in payloads if isinstance(p, str))
|
|
455
|
+
if doc is None:
|
|
456
|
+
return ContextItem(
|
|
457
|
+
id=hit.id or hit.url or hit.title,
|
|
458
|
+
source=_NAME,
|
|
459
|
+
kind=ItemKind.PAGE,
|
|
460
|
+
title=hit.title,
|
|
461
|
+
body=fallback_body,
|
|
462
|
+
url=hit.url,
|
|
463
|
+
)
|
|
464
|
+
metadata_raw = doc.get("metadata")
|
|
465
|
+
meta = cast(JsonObject, metadata_raw) if isinstance(metadata_raw, dict) else {}
|
|
466
|
+
return ContextItem(
|
|
467
|
+
id=opt_str(doc.get("id")) or hit.id or hit.url or hit.title,
|
|
468
|
+
source=_NAME,
|
|
469
|
+
kind=ItemKind.PAGE,
|
|
470
|
+
title=opt_str(doc.get("title")) or hit.title,
|
|
471
|
+
body=_first_str(doc, _BODY_KEYS) or fallback_body,
|
|
472
|
+
url=opt_str(doc.get("url")) or hit.url,
|
|
473
|
+
created_at=_timestamp(doc, meta, ("created_time", "created_at")),
|
|
474
|
+
updated_at=_timestamp(doc, meta, ("last_edited_time", "updated_at")),
|
|
475
|
+
metadata=meta,
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _timestamp(doc: JsonObject, meta: JsonObject, keys: tuple[str, ...]) -> str | None:
|
|
480
|
+
"""A timestamp under any of `keys`, checking the document first, then its metadata."""
|
|
481
|
+
return _first_str(doc, keys) or _first_str(meta, keys)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _within_window(item: ContextItem, since: datetime | None, until: datetime | None) -> bool:
|
|
485
|
+
"""Client-side since/until filter on `updated_at` (items without timestamps pass)."""
|
|
486
|
+
updated = _parse_iso(item.updated_at)
|
|
487
|
+
if updated is None:
|
|
488
|
+
return True
|
|
489
|
+
if since is not None and updated < since:
|
|
490
|
+
return False
|
|
491
|
+
return not (until is not None and updated > until)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
# -- REST helpers ---------------------------------------------------------------------------
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _raise_on_rest_error(response: httpx.Response, *, doing: str) -> None:
|
|
498
|
+
"""Map Notion REST failures to ConnectErrors that say how to recover."""
|
|
499
|
+
if response.is_success:
|
|
500
|
+
return
|
|
501
|
+
if response.status_code in (401, 403):
|
|
502
|
+
raise ConnectError(
|
|
503
|
+
f"notion rejected the credential (HTTP {response.status_code}) during {doing}; "
|
|
504
|
+
f"{_RECONNECT_HINT} or update ${token_env_var(_NAME)}"
|
|
505
|
+
)
|
|
506
|
+
raise ConnectError(
|
|
507
|
+
f"notion {doing} failed (HTTP {response.status_code}): {response.text[:200]}; "
|
|
508
|
+
f"retry, and {_RECONNECT_HINT} if it keeps failing"
|
|
509
|
+
)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _json_object(response: httpx.Response) -> JsonObject:
|
|
513
|
+
"""The response body as a JSON object, or a ConnectError naming the endpoint."""
|
|
514
|
+
try:
|
|
515
|
+
payload = response.json()
|
|
516
|
+
except ValueError:
|
|
517
|
+
payload = None
|
|
518
|
+
if not isinstance(payload, dict):
|
|
519
|
+
raise ConnectError(
|
|
520
|
+
f"notion returned a non-JSON-object body from {response.request.url.path}; "
|
|
521
|
+
"retry, and report a bug if it keeps happening"
|
|
522
|
+
)
|
|
523
|
+
return cast(JsonObject, payload)
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def _search_pages(client: httpx.Client, query: PullQuery) -> list[JsonObject]:
|
|
527
|
+
"""POST /v1/search for pages sorted by last_edited_time desc, up to `query.limit`.
|
|
528
|
+
|
|
529
|
+
`since`/`until` are applied client-side (the search API has no time filter); the
|
|
530
|
+
descending sort lets pagination stop at the first page older than `since`.
|
|
531
|
+
"""
|
|
532
|
+
since = _parse_iso(query.since)
|
|
533
|
+
until = _parse_iso(query.until)
|
|
534
|
+
pages: list[JsonObject] = []
|
|
535
|
+
cursor: str | None = None
|
|
536
|
+
while True:
|
|
537
|
+
body: JsonObject = {
|
|
538
|
+
"page_size": min(query.limit, _PAGE_SIZE),
|
|
539
|
+
"sort": {"direction": "descending", "timestamp": "last_edited_time"},
|
|
540
|
+
"filter": {"property": "object", "value": "page"},
|
|
541
|
+
}
|
|
542
|
+
if query.query:
|
|
543
|
+
body["query"] = query.query
|
|
544
|
+
if cursor:
|
|
545
|
+
body["start_cursor"] = cursor
|
|
546
|
+
response = client.post("/v1/search", json=body)
|
|
547
|
+
_raise_on_rest_error(response, doing="search")
|
|
548
|
+
payload = _json_object(response)
|
|
549
|
+
results = payload.get("results")
|
|
550
|
+
for result in results if isinstance(results, list) else []:
|
|
551
|
+
if not isinstance(result, dict) or result.get("object") != "page":
|
|
552
|
+
continue
|
|
553
|
+
edited = _parse_iso(opt_str(result.get("last_edited_time")))
|
|
554
|
+
if since is not None and edited is not None and edited < since:
|
|
555
|
+
return pages # descending sort: everything after this is older still
|
|
556
|
+
if until is not None and edited is not None and edited > until:
|
|
557
|
+
continue
|
|
558
|
+
pages.append(result)
|
|
559
|
+
if len(pages) >= query.limit:
|
|
560
|
+
return pages
|
|
561
|
+
cursor = opt_str(payload.get("next_cursor")) if payload.get("has_more") else None
|
|
562
|
+
if cursor is None:
|
|
563
|
+
return pages
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _block_lines(client: httpx.Client, block_id: str, *, depth: int) -> list[str]:
|
|
567
|
+
"""Markdown lines for one block's children (paginated; recurses `_MAX_BLOCK_DEPTH` levels)."""
|
|
568
|
+
lines: list[str] = []
|
|
569
|
+
cursor: str | None = None
|
|
570
|
+
while True:
|
|
571
|
+
params: dict[str, str | int] = {"page_size": _PAGE_SIZE}
|
|
572
|
+
if cursor:
|
|
573
|
+
params["start_cursor"] = cursor
|
|
574
|
+
response = client.get(f"/v1/blocks/{block_id}/children", params=params)
|
|
575
|
+
_raise_on_rest_error(response, doing="block fetch")
|
|
576
|
+
payload = _json_object(response)
|
|
577
|
+
results = payload.get("results")
|
|
578
|
+
for block in results if isinstance(results, list) else []:
|
|
579
|
+
if not isinstance(block, dict):
|
|
580
|
+
continue
|
|
581
|
+
line = _block_line(block)
|
|
582
|
+
if line is not None:
|
|
583
|
+
lines.append(line)
|
|
584
|
+
child_id = opt_str(block.get("id"))
|
|
585
|
+
if depth < _MAX_BLOCK_DEPTH and block.get("has_children") is True and child_id:
|
|
586
|
+
child_lines = _block_lines(client, child_id, depth=depth + 1)
|
|
587
|
+
lines.extend(f" {child}" for child in child_lines)
|
|
588
|
+
cursor = opt_str(payload.get("next_cursor")) if payload.get("has_more") else None
|
|
589
|
+
if cursor is None:
|
|
590
|
+
return lines
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
_HEADING_PREFIXES = {"heading_1": "# ", "heading_2": "## ", "heading_3": "### "}
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _block_line(block: JsonObject) -> str | None:
|
|
597
|
+
"""One block flattened to a markdown line (None for empty or unsupported block types)."""
|
|
598
|
+
block_type = block.get("type")
|
|
599
|
+
if not isinstance(block_type, str):
|
|
600
|
+
return None
|
|
601
|
+
data = block.get(block_type)
|
|
602
|
+
if not isinstance(data, dict):
|
|
603
|
+
return None
|
|
604
|
+
text = _plain_text(data.get("rich_text"))
|
|
605
|
+
if block_type == "paragraph":
|
|
606
|
+
return text or None
|
|
607
|
+
if block_type in _HEADING_PREFIXES:
|
|
608
|
+
return f"{_HEADING_PREFIXES[block_type]}{text}" if text else None
|
|
609
|
+
if block_type == "bulleted_list_item":
|
|
610
|
+
return f"- {text}"
|
|
611
|
+
if block_type == "numbered_list_item":
|
|
612
|
+
return f"1. {text}"
|
|
613
|
+
if block_type == "to_do":
|
|
614
|
+
marker = "x" if data.get("checked") else " "
|
|
615
|
+
return f"- [{marker}] {text}"
|
|
616
|
+
if block_type in ("quote", "callout"):
|
|
617
|
+
return f"> {text}" if text else None
|
|
618
|
+
if block_type == "code":
|
|
619
|
+
language = data.get("language")
|
|
620
|
+
fence = language if isinstance(language, str) else ""
|
|
621
|
+
return f"```{fence}\n{text}\n```"
|
|
622
|
+
return None
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _plain_text(rich_text: JsonValue | None) -> str:
|
|
626
|
+
"""The concatenated `plain_text` spans of a Notion rich-text array."""
|
|
627
|
+
if not isinstance(rich_text, list):
|
|
628
|
+
return ""
|
|
629
|
+
parts: list[str] = []
|
|
630
|
+
for span in rich_text:
|
|
631
|
+
if isinstance(span, dict):
|
|
632
|
+
text = span.get("plain_text")
|
|
633
|
+
if isinstance(text, str):
|
|
634
|
+
parts.append(text)
|
|
635
|
+
return "".join(parts)
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def _page_title(page: JsonObject) -> str:
|
|
639
|
+
"""The page's title property rendered to plain text ("Untitled" when empty)."""
|
|
640
|
+
properties = page.get("properties")
|
|
641
|
+
if isinstance(properties, dict):
|
|
642
|
+
for prop in properties.values():
|
|
643
|
+
if isinstance(prop, dict) and prop.get("type") == "title":
|
|
644
|
+
title = _plain_text(prop.get("title"))
|
|
645
|
+
if title:
|
|
646
|
+
return title
|
|
647
|
+
return "Untitled"
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
def _page_item(page: JsonObject, body: str) -> ContextItem:
|
|
651
|
+
"""Normalize one search-result page object plus its rendered block body."""
|
|
652
|
+
metadata: JsonObject = {}
|
|
653
|
+
if "archived" in page:
|
|
654
|
+
metadata["archived"] = page.get("archived")
|
|
655
|
+
return ContextItem(
|
|
656
|
+
id=opt_str(page.get("id")) or "",
|
|
657
|
+
source=_NAME,
|
|
658
|
+
kind=ItemKind.PAGE,
|
|
659
|
+
title=_page_title(page),
|
|
660
|
+
body=body,
|
|
661
|
+
url=opt_str(page.get("url")),
|
|
662
|
+
created_at=opt_str(page.get("created_time")),
|
|
663
|
+
updated_at=opt_str(page.get("last_edited_time")),
|
|
664
|
+
metadata=metadata,
|
|
665
|
+
)
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
# -- the connector ----------------------------------------------------------------------------
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
class NotionConnector:
|
|
672
|
+
"""Notion connector: MCP OAuth by default, an internal-integration token as fallback."""
|
|
673
|
+
|
|
674
|
+
name = _NAME
|
|
675
|
+
label = "Notion"
|
|
676
|
+
|
|
677
|
+
def __init__(self, transport: httpx.BaseTransport | None = None) -> None:
|
|
678
|
+
"""`transport` is injected into every REST httpx call (None = the real network)."""
|
|
679
|
+
self._transport = transport
|
|
680
|
+
|
|
681
|
+
def connect(self, ui: ConnectUI) -> ConnectorAuth:
|
|
682
|
+
"""Interactive auth: env token, then pasted integration secret, then browser MCP OAuth."""
|
|
683
|
+
env_var = token_env_var(self.name)
|
|
684
|
+
env_token = (os.environ.get(env_var) or "").strip()
|
|
685
|
+
if env_token:
|
|
686
|
+
auth = ConnectorAuth(kind="token", access_token=env_token)
|
|
687
|
+
account = self._verify_rest(auth)
|
|
688
|
+
ui.info(f"using the ${env_var} integration token ({account})")
|
|
689
|
+
return auth.model_copy(update={"account": account})
|
|
690
|
+
secret = ui.prompt_secret(
|
|
691
|
+
"Notion internal integration secret (press Enter to use browser OAuth instead)"
|
|
692
|
+
).strip()
|
|
693
|
+
if secret:
|
|
694
|
+
auth = ConnectorAuth(kind="token", access_token=secret)
|
|
695
|
+
account = self._verify_rest(auth)
|
|
696
|
+
ui.info(f"integration token verified ({account})")
|
|
697
|
+
return auth.model_copy(update={"account": account})
|
|
698
|
+
ui.info("authorizing with the hosted Notion MCP server (mcp.notion.com)")
|
|
699
|
+
return self._connect_mcp(ui)
|
|
700
|
+
|
|
701
|
+
def verify(self, auth: ConnectorAuth) -> str:
|
|
702
|
+
"""Cheapest identity call: REST GET /v1/users/me, or MCP initialize + list_tools."""
|
|
703
|
+
if auth.kind == "token":
|
|
704
|
+
return self._verify_rest(auth)
|
|
705
|
+
return asyncio.run(self._verify_mcp())
|
|
706
|
+
|
|
707
|
+
def pull(self, auth: ConnectorAuth, query: PullQuery) -> list[ContextItem]:
|
|
708
|
+
"""Fetch pages matching `query`, normalized and capped at `query.limit`."""
|
|
709
|
+
if query.limit <= 0:
|
|
710
|
+
return []
|
|
711
|
+
if auth.kind == "token":
|
|
712
|
+
return self._pull_rest(auth, query)
|
|
713
|
+
return asyncio.run(self._pull_mcp(query))
|
|
714
|
+
|
|
715
|
+
# -- MCP path ------------------------------------------------------------------------
|
|
716
|
+
|
|
717
|
+
def _connect_mcp(self, ui: ConnectUI) -> ConnectorAuth:
|
|
718
|
+
"""Run the browser MCP OAuth flow against a single-use localhost redirect."""
|
|
719
|
+
server = LoopbackServer()
|
|
720
|
+
deadline = time.monotonic() + _OAUTH_TIMEOUT_SECONDS
|
|
721
|
+
thread = threading.Thread(
|
|
722
|
+
target=serve_until, args=(server, deadline), name="wmo-notion-oauth", daemon=True
|
|
723
|
+
)
|
|
724
|
+
port = int(server.server_address[1])
|
|
725
|
+
redirect_uri = f"http://127.0.0.1:{port}/callback"
|
|
726
|
+
try:
|
|
727
|
+
thread.start()
|
|
728
|
+
account = asyncio.run(
|
|
729
|
+
_authorize_and_identify(ui, server, redirect_uri, _OAUTH_TIMEOUT_SECONDS)
|
|
730
|
+
)
|
|
731
|
+
finally:
|
|
732
|
+
server.received.set()
|
|
733
|
+
thread.join(2.0)
|
|
734
|
+
server.server_close()
|
|
735
|
+
auth = _stored_oauth_auth()
|
|
736
|
+
if auth is None or not auth.access_token:
|
|
737
|
+
raise ConnectError(
|
|
738
|
+
f"the Notion MCP authorization finished without a stored token; {_RECONNECT_HINT}"
|
|
739
|
+
)
|
|
740
|
+
auth = auth.model_copy(update={"account": account})
|
|
741
|
+
save_connector_auth(self.name, auth)
|
|
742
|
+
return auth
|
|
743
|
+
|
|
744
|
+
async def _verify_mcp(self) -> str:
|
|
745
|
+
"""Open an MCP session (refreshing the token if needed) and count its tools."""
|
|
746
|
+
async with _mcp_session() as session:
|
|
747
|
+
return await _verify_via_session(session)
|
|
748
|
+
|
|
749
|
+
async def _pull_mcp(self, query: PullQuery) -> list[ContextItem]:
|
|
750
|
+
"""Open an MCP session (refreshing the token if needed) and run search-then-fetch."""
|
|
751
|
+
async with _mcp_session() as session:
|
|
752
|
+
return await _pull_via_session(session, query)
|
|
753
|
+
|
|
754
|
+
# -- REST path -----------------------------------------------------------------------
|
|
755
|
+
|
|
756
|
+
def _rest_client(self, auth: ConnectorAuth) -> httpx.Client:
|
|
757
|
+
"""A client for api.notion.com carrying the bearer token and API version."""
|
|
758
|
+
return httpx.Client(
|
|
759
|
+
base_url=_API_BASE,
|
|
760
|
+
headers={
|
|
761
|
+
"Authorization": f"Bearer {auth.access_token}",
|
|
762
|
+
"Notion-Version": _NOTION_VERSION,
|
|
763
|
+
},
|
|
764
|
+
timeout=_REST_TIMEOUT_SECONDS,
|
|
765
|
+
transport=self._transport,
|
|
766
|
+
)
|
|
767
|
+
|
|
768
|
+
def _verify_rest(self, auth: ConnectorAuth) -> str:
|
|
769
|
+
"""GET /v1/users/me: the integration's bot name and workspace."""
|
|
770
|
+
with self._rest_client(auth) as client, transport_errors(_API_HOST):
|
|
771
|
+
response = client.get("/v1/users/me")
|
|
772
|
+
_raise_on_rest_error(response, doing="identity check")
|
|
773
|
+
payload = _json_object(response)
|
|
774
|
+
name = opt_str(payload.get("name")) or "Notion integration"
|
|
775
|
+
bot = payload.get("bot")
|
|
776
|
+
workspace = opt_str(bot.get("workspace_name")) if isinstance(bot, dict) else None
|
|
777
|
+
return f"{name} ({workspace})" if workspace else name
|
|
778
|
+
|
|
779
|
+
def _pull_rest(self, auth: ConnectorAuth, query: PullQuery) -> list[ContextItem]:
|
|
780
|
+
"""Search pages, then flatten each page's blocks (one nesting level) to markdown."""
|
|
781
|
+
items: list[ContextItem] = []
|
|
782
|
+
with self._rest_client(auth) as client, transport_errors(_API_HOST):
|
|
783
|
+
for page in _search_pages(client, query):
|
|
784
|
+
page_id = opt_str(page.get("id"))
|
|
785
|
+
body = "\n".join(_block_lines(client, page_id, depth=0)) if page_id else ""
|
|
786
|
+
items.append(_page_item(page, body))
|
|
787
|
+
return items
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
register_connector(NotionConnector())
|