spot-sdk-python 1.2.1__tar.gz → 2.0.0b1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spot_sdk_python-2.0.0b1/PKG-INFO +117 -0
- spot_sdk_python-2.0.0b1/README.md +93 -0
- {spot_sdk_python-1.2.1 → spot_sdk_python-2.0.0b1}/pyproject.toml +11 -7
- spot_sdk_python-2.0.0b1/spot_sdk/__init__.py +118 -0
- spot_sdk_python-2.0.0b1/spot_sdk/app.py +332 -0
- spot_sdk_python-2.0.0b1/spot_sdk/clients.py +211 -0
- {spot_sdk_python-1.2.1 → spot_sdk_python-2.0.0b1}/spot_sdk/config.py +0 -8
- spot_sdk_python-2.0.0b1/spot_sdk/email.py +165 -0
- {spot_sdk_python-1.2.1 → spot_sdk_python-2.0.0b1}/spot_sdk/knowledge.py +113 -115
- spot_sdk_python-2.0.0b1/spot_sdk/llm.py +523 -0
- spot_sdk_python-2.0.0b1/spot_sdk/manifest.py +122 -0
- spot_sdk_python-2.0.0b1/spot_sdk/mime.py +2038 -0
- spot_sdk_python-2.0.0b1/spot_sdk/orchestrator.py +92 -0
- spot_sdk_python-2.0.0b1/spot_sdk/py.typed +0 -0
- spot_sdk_python-2.0.0b1/spot_sdk/retriever.py +108 -0
- spot_sdk_python-2.0.0b1/spot_sdk/settings.py +79 -0
- spot_sdk_python-2.0.0b1/spot_sdk/signals.py +42 -0
- spot_sdk_python-2.0.0b1/spot_sdk/testing/README.md +29 -0
- spot_sdk_python-2.0.0b1/spot_sdk/testing/__init__.py +17 -0
- spot_sdk_python-2.0.0b1/spot_sdk/testing/contract.py +51 -0
- {spot_sdk_python-1.2.1 → spot_sdk_python-2.0.0b1}/spot_sdk/testing/fake_knowledge_client.py +42 -18
- spot_sdk_python-2.0.0b1/spot_sdk/testing/fake_llm.py +70 -0
- spot_sdk_python-2.0.0b1/spot_sdk/testing/fake_spot.py +108 -0
- spot_sdk_python-2.0.0b1/spot_sdk/testing/views.py +47 -0
- spot_sdk_python-2.0.0b1/spot_sdk/verdict.py +53 -0
- {spot_sdk_python-1.2.1 → spot_sdk_python-2.0.0b1}/spot_sdk/workflow.py +79 -14
- spot_sdk_python-1.2.1/PKG-INFO +0 -353
- spot_sdk_python-1.2.1/README.md +0 -331
- spot_sdk_python-1.2.1/spot_sdk/__init__.py +0 -99
- spot_sdk_python-1.2.1/spot_sdk/analysis_context.py +0 -99
- spot_sdk_python-1.2.1/spot_sdk/analyzer.py +0 -106
- spot_sdk_python-1.2.1/spot_sdk/analyzer_base.py +0 -323
- spot_sdk_python-1.2.1/spot_sdk/api_gateway.py +0 -271
- spot_sdk_python-1.2.1/spot_sdk/config_client.py +0 -203
- spot_sdk_python-1.2.1/spot_sdk/config_helpers.py +0 -34
- spot_sdk_python-1.2.1/spot_sdk/email.py +0 -136
- spot_sdk_python-1.2.1/spot_sdk/ollama.py +0 -58
- spot_sdk_python-1.2.1/spot_sdk/orchestrator.py +0 -79
- spot_sdk_python-1.2.1/spot_sdk/plugin.py +0 -35
- spot_sdk_python-1.2.1/spot_sdk/results.py +0 -129
- spot_sdk_python-1.2.1/spot_sdk/retriever.py +0 -267
- spot_sdk_python-1.2.1/spot_sdk/settings_schema.py +0 -56
- spot_sdk_python-1.2.1/spot_sdk/testing/README.md +0 -83
- spot_sdk_python-1.2.1/spot_sdk/testing/__init__.py +0 -22
- spot_sdk_python-1.2.1/spot_sdk/testing/factories.py +0 -177
- spot_sdk_python-1.2.1/spot_sdk/threat_levels.py +0 -33
- {spot_sdk_python-1.2.1 → spot_sdk_python-2.0.0b1}/spot_sdk/errors.py +0 -0
- {spot_sdk_python-1.2.1 → spot_sdk_python-2.0.0b1}/spot_sdk/knowledge_tags.py +0 -0
- {spot_sdk_python-1.2.1 → spot_sdk_python-2.0.0b1}/spot_sdk/logging.py +0 -0
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: spot-sdk-python
|
|
3
|
+
Version: 2.0.0b1
|
|
4
|
+
Summary: SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Author: SPOT Project
|
|
7
|
+
Author-email: spot@sonn.lu
|
|
8
|
+
Requires-Python: >=3.11,<3.16
|
|
9
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
16
|
+
Requires-Dist: fastapi (>=0.115.3,<1)
|
|
17
|
+
Requires-Dist: httpx (>=0.25,<1)
|
|
18
|
+
Requires-Dist: pydantic (>=2.10,<3.0)
|
|
19
|
+
Requires-Dist: pydantic-settings (>=2.0.0,<3.0.0)
|
|
20
|
+
Requires-Dist: selectolax (>=1.0,<1.1)
|
|
21
|
+
Project-URL: Homepage, https://sonn.lu
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# spot-sdk-python
|
|
25
|
+
|
|
26
|
+
The SDK for writing plugins of SPOT, the Spear-Phishing Overwatching
|
|
27
|
+
Tool: analyzers, context providers, mail retrievers and responders.
|
|
28
|
+
|
|
29
|
+
SPOT runs each plugin as a container and calls its `/internal/*` routes
|
|
30
|
+
with the plugin's own token. This package holds what SPOT's core and its
|
|
31
|
+
plugins share:
|
|
32
|
+
|
|
33
|
+
- **the contract** (Pydantic models): the email view, verdicts and
|
|
34
|
+
signals, workflow policies, results, and the plugin manifest with its
|
|
35
|
+
OCI labels;
|
|
36
|
+
- **the parser** core runs on every message at intake
|
|
37
|
+
(`spot_sdk.mime.parse`), bounded against hostile input, which records
|
|
38
|
+
what mail clients may read differently as anomalies;
|
|
39
|
+
- **the plugin runtime**: `create_plugin_app` builds the FastAPI app with
|
|
40
|
+
the routes core calls, token-checked; `PluginSettings` reads the token
|
|
41
|
+
and secrets from `/run/secrets` and the settings from the environment;
|
|
42
|
+
- **the clients**: `SpotClient` (ingest, message content, results),
|
|
43
|
+
`KnowledgeClient` (the Knowledge Store) and `LLMClient`
|
|
44
|
+
(OpenAI-compatible chat models, with the email kept apart from the
|
|
45
|
+
instructions as untrusted data);
|
|
46
|
+
- **test helpers**: `view_from_eml`, `make_view`, `FakeSpot`,
|
|
47
|
+
`FakeKnowledgeClient`, `FakeLLM` and `assert_contract`.
|
|
48
|
+
|
|
49
|
+
## Install
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install spot-sdk-python==2.0.0b1
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Python 3.11 to 3.15. Version 2 implements plugin contract 2, for
|
|
56
|
+
SPOT 2.0.
|
|
57
|
+
|
|
58
|
+
## An analyzer
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from spot_sdk import (
|
|
62
|
+
AnalyzerVerdict,
|
|
63
|
+
AnalyzeRequest,
|
|
64
|
+
Egress,
|
|
65
|
+
Manifest,
|
|
66
|
+
PluginKind,
|
|
67
|
+
PluginSettings,
|
|
68
|
+
Signal,
|
|
69
|
+
VerdictClass,
|
|
70
|
+
create_plugin_app,
|
|
71
|
+
)
|
|
72
|
+
from spot_sdk.signals import AUTH_DMARC_FAIL
|
|
73
|
+
|
|
74
|
+
MANIFEST = Manifest(
|
|
75
|
+
id="analyzer-dmarc",
|
|
76
|
+
version="1.0.0",
|
|
77
|
+
kinds=[PluginKind.ANALYZER],
|
|
78
|
+
egress=Egress.NONE,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def analyze(request: AnalyzeRequest, settings: PluginSettings) -> AnalyzerVerdict:
|
|
83
|
+
failed = [
|
|
84
|
+
result
|
|
85
|
+
for result in request.email.authentication
|
|
86
|
+
if result.method == "dmarc" and result.result == "fail"
|
|
87
|
+
]
|
|
88
|
+
if not failed:
|
|
89
|
+
return AnalyzerVerdict(verdict=VerdictClass.BENIGN, score=10, confidence=60)
|
|
90
|
+
return AnalyzerVerdict(
|
|
91
|
+
verdict=VerdictClass.SUSPICIOUS,
|
|
92
|
+
score=70,
|
|
93
|
+
confidence=80,
|
|
94
|
+
signals=[Signal(name=AUTH_DMARC_FAIL, weight=60, evidence="dmarc=fail")],
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
app = create_plugin_app(MANIFEST, PluginSettings, analyze=analyze)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Serve it with `uvicorn analyzer_dmarc:app --host 0.0.0.0 --port 8000`,
|
|
102
|
+
and test it with `spot_sdk.testing.assert_contract(app, "tests/phish.eml",
|
|
103
|
+
token=...)`.
|
|
104
|
+
|
|
105
|
+
## Documentation
|
|
106
|
+
|
|
107
|
+
- [Documentation](https://spot-project.codeberg.page/documentation/sdk/):
|
|
108
|
+
the contract, writing each kind of plugin, the reference.
|
|
109
|
+
- [Migrating from 1.x](https://spot-project.codeberg.page/documentation/sdk/MIGRATION-2/):
|
|
110
|
+
every removed name and its replacement. SDK 2 has no compatibility
|
|
111
|
+
with contract 1.
|
|
112
|
+
- [Source and changelog](https://codeberg.org/SPOT_Project/sdk).
|
|
113
|
+
|
|
114
|
+
## License
|
|
115
|
+
|
|
116
|
+
Apache License 2.0.
|
|
117
|
+
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# spot-sdk-python
|
|
2
|
+
|
|
3
|
+
The SDK for writing plugins of SPOT, the Spear-Phishing Overwatching
|
|
4
|
+
Tool: analyzers, context providers, mail retrievers and responders.
|
|
5
|
+
|
|
6
|
+
SPOT runs each plugin as a container and calls its `/internal/*` routes
|
|
7
|
+
with the plugin's own token. This package holds what SPOT's core and its
|
|
8
|
+
plugins share:
|
|
9
|
+
|
|
10
|
+
- **the contract** (Pydantic models): the email view, verdicts and
|
|
11
|
+
signals, workflow policies, results, and the plugin manifest with its
|
|
12
|
+
OCI labels;
|
|
13
|
+
- **the parser** core runs on every message at intake
|
|
14
|
+
(`spot_sdk.mime.parse`), bounded against hostile input, which records
|
|
15
|
+
what mail clients may read differently as anomalies;
|
|
16
|
+
- **the plugin runtime**: `create_plugin_app` builds the FastAPI app with
|
|
17
|
+
the routes core calls, token-checked; `PluginSettings` reads the token
|
|
18
|
+
and secrets from `/run/secrets` and the settings from the environment;
|
|
19
|
+
- **the clients**: `SpotClient` (ingest, message content, results),
|
|
20
|
+
`KnowledgeClient` (the Knowledge Store) and `LLMClient`
|
|
21
|
+
(OpenAI-compatible chat models, with the email kept apart from the
|
|
22
|
+
instructions as untrusted data);
|
|
23
|
+
- **test helpers**: `view_from_eml`, `make_view`, `FakeSpot`,
|
|
24
|
+
`FakeKnowledgeClient`, `FakeLLM` and `assert_contract`.
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install spot-sdk-python==2.0.0b1
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Python 3.11 to 3.15. Version 2 implements plugin contract 2, for
|
|
33
|
+
SPOT 2.0.
|
|
34
|
+
|
|
35
|
+
## An analyzer
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from spot_sdk import (
|
|
39
|
+
AnalyzerVerdict,
|
|
40
|
+
AnalyzeRequest,
|
|
41
|
+
Egress,
|
|
42
|
+
Manifest,
|
|
43
|
+
PluginKind,
|
|
44
|
+
PluginSettings,
|
|
45
|
+
Signal,
|
|
46
|
+
VerdictClass,
|
|
47
|
+
create_plugin_app,
|
|
48
|
+
)
|
|
49
|
+
from spot_sdk.signals import AUTH_DMARC_FAIL
|
|
50
|
+
|
|
51
|
+
MANIFEST = Manifest(
|
|
52
|
+
id="analyzer-dmarc",
|
|
53
|
+
version="1.0.0",
|
|
54
|
+
kinds=[PluginKind.ANALYZER],
|
|
55
|
+
egress=Egress.NONE,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def analyze(request: AnalyzeRequest, settings: PluginSettings) -> AnalyzerVerdict:
|
|
60
|
+
failed = [
|
|
61
|
+
result
|
|
62
|
+
for result in request.email.authentication
|
|
63
|
+
if result.method == "dmarc" and result.result == "fail"
|
|
64
|
+
]
|
|
65
|
+
if not failed:
|
|
66
|
+
return AnalyzerVerdict(verdict=VerdictClass.BENIGN, score=10, confidence=60)
|
|
67
|
+
return AnalyzerVerdict(
|
|
68
|
+
verdict=VerdictClass.SUSPICIOUS,
|
|
69
|
+
score=70,
|
|
70
|
+
confidence=80,
|
|
71
|
+
signals=[Signal(name=AUTH_DMARC_FAIL, weight=60, evidence="dmarc=fail")],
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
app = create_plugin_app(MANIFEST, PluginSettings, analyze=analyze)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Serve it with `uvicorn analyzer_dmarc:app --host 0.0.0.0 --port 8000`,
|
|
79
|
+
and test it with `spot_sdk.testing.assert_contract(app, "tests/phish.eml",
|
|
80
|
+
token=...)`.
|
|
81
|
+
|
|
82
|
+
## Documentation
|
|
83
|
+
|
|
84
|
+
- [Documentation](https://spot-project.codeberg.page/documentation/sdk/):
|
|
85
|
+
the contract, writing each kind of plugin, the reference.
|
|
86
|
+
- [Migrating from 1.x](https://spot-project.codeberg.page/documentation/sdk/MIGRATION-2/):
|
|
87
|
+
every removed name and its replacement. SDK 2 has no compatibility
|
|
88
|
+
with contract 1.
|
|
89
|
+
- [Source and changelog](https://codeberg.org/SPOT_Project/sdk).
|
|
90
|
+
|
|
91
|
+
## License
|
|
92
|
+
|
|
93
|
+
Apache License 2.0.
|
|
@@ -4,8 +4,8 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "spot-sdk-python"
|
|
7
|
-
version = "
|
|
8
|
-
description = "
|
|
7
|
+
version = "2.0.0b1"
|
|
8
|
+
description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
|
|
9
9
|
authors = ["SPOT Project <spot@sonn.lu>"]
|
|
10
10
|
license = "Apache-2.0"
|
|
11
11
|
homepage = "https://sonn.lu"
|
|
@@ -15,18 +15,22 @@ packages = [
|
|
|
15
15
|
]
|
|
16
16
|
|
|
17
17
|
[tool.poetry.dependencies]
|
|
18
|
-
python = "
|
|
19
|
-
pydantic =
|
|
18
|
+
python = ">=3.11,<3.16"
|
|
19
|
+
pydantic = "^2.10"
|
|
20
20
|
pydantic-settings = "^2.0.0"
|
|
21
|
-
httpx = "
|
|
22
|
-
|
|
21
|
+
httpx = ">=0.25,<1"
|
|
22
|
+
selectolax = ">=1.0,<1.1"
|
|
23
|
+
fastapi = ">=0.115.3,<1"
|
|
23
24
|
|
|
24
25
|
[tool.poetry.group.dev.dependencies]
|
|
25
26
|
pytest = "^7.4.0"
|
|
26
27
|
pytest-asyncio = "^0.21.0"
|
|
28
|
+
pytest-cov = "^4.1.0"
|
|
27
29
|
mypy = "^1.5.0"
|
|
28
30
|
|
|
29
31
|
[tool.ruff]
|
|
32
|
+
target-version = "py311"
|
|
33
|
+
|
|
34
|
+
[tool.ruff.lint]
|
|
30
35
|
select = ["E", "F", "I", "N", "W"]
|
|
31
36
|
ignore = ["E501"] # Line too long
|
|
32
|
-
target-version = "py311"
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""SPOT SDK: the plugin contract v2, the plugin runtime and its clients."""
|
|
2
|
+
|
|
3
|
+
from .app import SyncState, create_plugin_app
|
|
4
|
+
from .clients import SpotAPIError, SpotClient
|
|
5
|
+
from .config import ConfigOption, ConfigStatus, ConfigUpdateRequest
|
|
6
|
+
from .email import Address, Anomaly, AuthResult, EmailView, Envelope, Header, Part, Url
|
|
7
|
+
from .errors import ErrorResponse
|
|
8
|
+
from .knowledge import KnowledgeClient, KnowledgeDocument, chunk_text, content_hash
|
|
9
|
+
from .knowledge_tags import KnowledgeTag
|
|
10
|
+
from .llm import LLMClient, LLMOutputError, LLMToolCall, LLMToolCalls, LLMToolRound
|
|
11
|
+
from .logging import configure_logging, get_logger, get_logging_config
|
|
12
|
+
from .manifest import (
|
|
13
|
+
CONTRACT_VERSION,
|
|
14
|
+
Egress,
|
|
15
|
+
Manifest,
|
|
16
|
+
PluginKind,
|
|
17
|
+
Resources,
|
|
18
|
+
SettingField,
|
|
19
|
+
SettingType,
|
|
20
|
+
)
|
|
21
|
+
from .orchestrator import (
|
|
22
|
+
AnalyzeRequest,
|
|
23
|
+
AnalyzerOutcome,
|
|
24
|
+
AttributedSignal,
|
|
25
|
+
OrchestrationResult,
|
|
26
|
+
RespondRequest,
|
|
27
|
+
StageOutcome,
|
|
28
|
+
)
|
|
29
|
+
from .retriever import IngestionRequest, IngestionResult, IngestionStatus
|
|
30
|
+
from .settings import PluginSettings
|
|
31
|
+
from .verdict import AnalyzerVerdict, Score, Signal, VerdictClass
|
|
32
|
+
from .workflow import (
|
|
33
|
+
AggregationMethod,
|
|
34
|
+
AnalyzerConfig,
|
|
35
|
+
DecisiveSignal,
|
|
36
|
+
FailureStrategy,
|
|
37
|
+
ResponseMode,
|
|
38
|
+
RetrievalLimits,
|
|
39
|
+
RetryConfig,
|
|
40
|
+
VerdictPolicy,
|
|
41
|
+
Workflow,
|
|
42
|
+
WorkflowStage,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
__all__ = [
|
|
46
|
+
# Email view
|
|
47
|
+
"Address",
|
|
48
|
+
"Anomaly",
|
|
49
|
+
"AuthResult",
|
|
50
|
+
"EmailView",
|
|
51
|
+
"Envelope",
|
|
52
|
+
"Header",
|
|
53
|
+
"Part",
|
|
54
|
+
"Url",
|
|
55
|
+
# Verdicts
|
|
56
|
+
"AnalyzerVerdict",
|
|
57
|
+
"Score",
|
|
58
|
+
"Signal",
|
|
59
|
+
"VerdictClass",
|
|
60
|
+
# Workflow
|
|
61
|
+
"AggregationMethod",
|
|
62
|
+
"AnalyzerConfig",
|
|
63
|
+
"DecisiveSignal",
|
|
64
|
+
"FailureStrategy",
|
|
65
|
+
"ResponseMode",
|
|
66
|
+
"RetrievalLimits",
|
|
67
|
+
"RetryConfig",
|
|
68
|
+
"VerdictPolicy",
|
|
69
|
+
"Workflow",
|
|
70
|
+
"WorkflowStage",
|
|
71
|
+
# Analyzer and responder calls, results
|
|
72
|
+
"AnalyzeRequest",
|
|
73
|
+
"AnalyzerOutcome",
|
|
74
|
+
"AttributedSignal",
|
|
75
|
+
"OrchestrationResult",
|
|
76
|
+
"RespondRequest",
|
|
77
|
+
"StageOutcome",
|
|
78
|
+
# Ingestion
|
|
79
|
+
"IngestionRequest",
|
|
80
|
+
"IngestionResult",
|
|
81
|
+
"IngestionStatus",
|
|
82
|
+
# Plugin manifest
|
|
83
|
+
"CONTRACT_VERSION",
|
|
84
|
+
"Egress",
|
|
85
|
+
"Manifest",
|
|
86
|
+
"PluginKind",
|
|
87
|
+
"Resources",
|
|
88
|
+
"SettingField",
|
|
89
|
+
"SettingType",
|
|
90
|
+
# Plugin runtime
|
|
91
|
+
"PluginSettings",
|
|
92
|
+
"SyncState",
|
|
93
|
+
"create_plugin_app",
|
|
94
|
+
# Clients
|
|
95
|
+
"LLMClient",
|
|
96
|
+
"LLMOutputError",
|
|
97
|
+
"LLMToolCall",
|
|
98
|
+
"LLMToolCalls",
|
|
99
|
+
"LLMToolRound",
|
|
100
|
+
"SpotAPIError",
|
|
101
|
+
"SpotClient",
|
|
102
|
+
# Knowledge Store
|
|
103
|
+
"KnowledgeClient",
|
|
104
|
+
"KnowledgeDocument",
|
|
105
|
+
"KnowledgeTag",
|
|
106
|
+
"chunk_text",
|
|
107
|
+
"content_hash",
|
|
108
|
+
# Config models
|
|
109
|
+
"ConfigOption",
|
|
110
|
+
"ConfigStatus",
|
|
111
|
+
"ConfigUpdateRequest",
|
|
112
|
+
# Error response
|
|
113
|
+
"ErrorResponse",
|
|
114
|
+
# Logging
|
|
115
|
+
"configure_logging",
|
|
116
|
+
"get_logger",
|
|
117
|
+
"get_logging_config",
|
|
118
|
+
]
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
"""The plugin runtime: one FastAPI app with the routes core calls.
|
|
2
|
+
|
|
3
|
+
``create_plugin_app`` builds it from the plugin's manifest, its settings
|
|
4
|
+
class and its handlers:
|
|
5
|
+
|
|
6
|
+
- ``GET /health`` and ``GET /manifest`` are open;
|
|
7
|
+
- ``POST /internal/analyze``, ``POST /internal/respond``, and
|
|
8
|
+
``POST /internal/sync`` with ``GET /internal/state``, exist for the
|
|
9
|
+
handlers given. Each checks ``Authorization: Bearer <the plugin's own
|
|
10
|
+
token>`` before it reads the request, and answers 401 otherwise; an empty
|
|
11
|
+
configured token refuses every call.
|
|
12
|
+
|
|
13
|
+
An ``async def`` handler, or an object whose ``__call__`` is ``async def``,
|
|
14
|
+
runs on the event loop; any other callable runs in a worker thread, so it may
|
|
15
|
+
block. Either way the clients send the call's ``X-Correlation-ID`` on to SPOT.
|
|
16
|
+
|
|
17
|
+
Neither the logs nor the error answers hold the request's content: a
|
|
18
|
+
handler's exception is logged by its type and place only, and an invalid
|
|
19
|
+
request is answered with the fields refused, never the values sent.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import asyncio
|
|
25
|
+
import hmac
|
|
26
|
+
import inspect
|
|
27
|
+
import traceback
|
|
28
|
+
from collections.abc import Awaitable, Callable, Iterator
|
|
29
|
+
from contextlib import contextmanager
|
|
30
|
+
from datetime import UTC, datetime
|
|
31
|
+
from typing import Any, TypeAlias, TypeVar
|
|
32
|
+
|
|
33
|
+
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
|
|
34
|
+
from fastapi.responses import JSONResponse
|
|
35
|
+
from pydantic import BaseModel, Field, SecretStr, ValidationError
|
|
36
|
+
from pydantic_core import PydanticSerializationError
|
|
37
|
+
from starlette.concurrency import run_in_threadpool
|
|
38
|
+
from starlette.types import Lifespan
|
|
39
|
+
|
|
40
|
+
from .clients import correlation_id
|
|
41
|
+
from .logging import get_logger
|
|
42
|
+
from .manifest import Manifest, PluginKind
|
|
43
|
+
from .orchestrator import AnalyzeRequest, RespondRequest
|
|
44
|
+
from .settings import PluginSettings
|
|
45
|
+
from .verdict import AnalyzerVerdict
|
|
46
|
+
|
|
47
|
+
logger = get_logger(__name__)
|
|
48
|
+
|
|
49
|
+
SettingsT = TypeVar("SettingsT", bound=PluginSettings)
|
|
50
|
+
ModelT = TypeVar("ModelT", bound=BaseModel)
|
|
51
|
+
|
|
52
|
+
AnalyzeHandler: TypeAlias = Callable[
|
|
53
|
+
[AnalyzeRequest, SettingsT], AnalyzerVerdict | Awaitable[AnalyzerVerdict]
|
|
54
|
+
]
|
|
55
|
+
"""``analyze(request, settings) -> AnalyzerVerdict``; an exception is answered 500."""
|
|
56
|
+
RespondHandler: TypeAlias = Callable[
|
|
57
|
+
[RespondRequest, SettingsT], None | Awaitable[None]
|
|
58
|
+
]
|
|
59
|
+
"""``respond(request, settings) -> None``; it acts only when ``request.mode`` is enforce."""
|
|
60
|
+
SyncHandler: TypeAlias = Callable[[SettingsT], int | Awaitable[int]]
|
|
61
|
+
"""``sync(settings) -> int``: the number of documents written."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class SyncState(BaseModel):
|
|
65
|
+
"""A context provider's syncs, as ``/internal/sync`` and ``/internal/state`` answer."""
|
|
66
|
+
|
|
67
|
+
in_progress: bool = False
|
|
68
|
+
last_sync_at: datetime | None = Field(
|
|
69
|
+
default=None, description="When the last sync started"
|
|
70
|
+
)
|
|
71
|
+
last_finished_at: datetime | None = Field(
|
|
72
|
+
default=None, description="When the last sync ended"
|
|
73
|
+
)
|
|
74
|
+
last_sync_count: int = Field(
|
|
75
|
+
default=0, description="Documents written by the last sync that succeeded"
|
|
76
|
+
)
|
|
77
|
+
last_error: str | None = Field(
|
|
78
|
+
default=None,
|
|
79
|
+
description="The exception's type when the last sync failed; None when it succeeded",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def create_plugin_app(
|
|
84
|
+
manifest: Manifest,
|
|
85
|
+
settings_cls: type[SettingsT],
|
|
86
|
+
*,
|
|
87
|
+
analyze: AnalyzeHandler[SettingsT] | None = None,
|
|
88
|
+
respond: RespondHandler[SettingsT] | None = None,
|
|
89
|
+
sync: SyncHandler[SettingsT] | None = None,
|
|
90
|
+
lifespan: Lifespan[FastAPI] | None = None,
|
|
91
|
+
) -> FastAPI:
|
|
92
|
+
"""Build the plugin's app; the settings are read once, here.
|
|
93
|
+
|
|
94
|
+
Each kind the manifest declares needs its handler (analyzer: ``analyze``,
|
|
95
|
+
responder: ``respond``, context provider: ``sync``; a mail retriever has
|
|
96
|
+
none), and each handler its kind, or ``ValueError`` is raised.
|
|
97
|
+
|
|
98
|
+
``lifespan`` is passed to FastAPI as is: an async context manager run
|
|
99
|
+
around the app's life, to load a model at startup, say.
|
|
100
|
+
|
|
101
|
+
``POST /internal/sync`` answers 202 at once and runs ``sync`` in the
|
|
102
|
+
background. While a sync runs, another call starts nothing and answers
|
|
103
|
+
202 with the running sync's state.
|
|
104
|
+
"""
|
|
105
|
+
_check_kinds(manifest, {"analyze": analyze, "respond": respond, "sync": sync})
|
|
106
|
+
settings = settings_cls()
|
|
107
|
+
if not settings.spot_plugin_token.get_secret_value():
|
|
108
|
+
logger.warning(
|
|
109
|
+
"The plugin has no token: every /internal call will be refused. Core "
|
|
110
|
+
"mounts it as /run/secrets/spot_plugin_token; outside a container, set "
|
|
111
|
+
"SPOT_PLUGIN_TOKEN."
|
|
112
|
+
)
|
|
113
|
+
app = FastAPI(
|
|
114
|
+
title=manifest.id,
|
|
115
|
+
version=manifest.version,
|
|
116
|
+
openapi_url=None,
|
|
117
|
+
lifespan=lifespan,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
@app.get("/health")
|
|
121
|
+
async def health() -> dict[str, str]:
|
|
122
|
+
return {"status": "healthy", "version": manifest.version}
|
|
123
|
+
|
|
124
|
+
@app.get("/manifest")
|
|
125
|
+
async def get_manifest() -> Manifest:
|
|
126
|
+
return manifest
|
|
127
|
+
|
|
128
|
+
internal = APIRouter(
|
|
129
|
+
prefix="/internal",
|
|
130
|
+
dependencies=[Depends(_token_check(settings.spot_plugin_token))],
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
if analyze is not None:
|
|
134
|
+
|
|
135
|
+
@internal.post("/analyze")
|
|
136
|
+
async def analyze_route(request: Request) -> Response:
|
|
137
|
+
call = await _parse(request, AnalyzeRequest)
|
|
138
|
+
with _correlated(request):
|
|
139
|
+
try:
|
|
140
|
+
answer = await _run(analyze, call, settings)
|
|
141
|
+
except Exception as error:
|
|
142
|
+
_log_failure(f"analysis of job {call.job_id}", error)
|
|
143
|
+
return JSONResponse({"detail": "analysis failed"}, status_code=500)
|
|
144
|
+
try:
|
|
145
|
+
verdict = AnalyzerVerdict.model_validate(answer)
|
|
146
|
+
except ValidationError as error:
|
|
147
|
+
_log_invalid_answer(f"analysis of job {call.job_id}", error)
|
|
148
|
+
return JSONResponse({"detail": "analysis failed"}, status_code=500)
|
|
149
|
+
try:
|
|
150
|
+
body = verdict.model_dump_json()
|
|
151
|
+
except PydanticSerializationError as error:
|
|
152
|
+
_log_failure(f"analysis of job {call.job_id}", error)
|
|
153
|
+
return JSONResponse({"detail": "analysis failed"}, status_code=500)
|
|
154
|
+
return Response(body, media_type="application/json")
|
|
155
|
+
|
|
156
|
+
if respond is not None:
|
|
157
|
+
|
|
158
|
+
@internal.post("/respond")
|
|
159
|
+
async def respond_route(request: Request) -> Response:
|
|
160
|
+
call = await _parse(request, RespondRequest)
|
|
161
|
+
with _correlated(request):
|
|
162
|
+
try:
|
|
163
|
+
await _run(respond, call, settings)
|
|
164
|
+
except Exception as error:
|
|
165
|
+
_log_failure(f"response to job {call.job_id}", error)
|
|
166
|
+
return JSONResponse({"detail": "response failed"}, status_code=500)
|
|
167
|
+
return Response(status_code=204)
|
|
168
|
+
|
|
169
|
+
if sync is not None:
|
|
170
|
+
syncs = _SyncRunner(sync, settings)
|
|
171
|
+
|
|
172
|
+
@internal.post("/sync", status_code=202)
|
|
173
|
+
async def sync_route(request: Request) -> SyncState:
|
|
174
|
+
with _correlated(request):
|
|
175
|
+
return syncs.start()
|
|
176
|
+
|
|
177
|
+
@internal.get("/state")
|
|
178
|
+
async def state_route() -> SyncState:
|
|
179
|
+
return syncs.state.model_copy()
|
|
180
|
+
|
|
181
|
+
app.include_router(internal)
|
|
182
|
+
return app
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
_HANDLER_KINDS = {
|
|
186
|
+
"analyze": PluginKind.ANALYZER,
|
|
187
|
+
"respond": PluginKind.RESPONDER,
|
|
188
|
+
"sync": PluginKind.CONTEXT_PROVIDER,
|
|
189
|
+
}
|
|
190
|
+
"""The kind each handler serves; a mail retriever has no handler."""
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _check_kinds(manifest: Manifest, handlers: dict[str, object]) -> None:
|
|
194
|
+
"""Each kind the manifest declares has its handler, and each handler its kind."""
|
|
195
|
+
for name, kind in _HANDLER_KINDS.items():
|
|
196
|
+
declared = kind in manifest.kinds
|
|
197
|
+
given = handlers[name] is not None
|
|
198
|
+
if declared and not given:
|
|
199
|
+
raise ValueError(
|
|
200
|
+
f"the manifest declares {kind}, but no {name} handler is given"
|
|
201
|
+
)
|
|
202
|
+
if given and not declared:
|
|
203
|
+
raise ValueError(
|
|
204
|
+
f"the {name} handler is given, but the manifest does not declare {kind}"
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class _SyncRunner:
|
|
209
|
+
"""Runs the sync handler in the background, one run at a time."""
|
|
210
|
+
|
|
211
|
+
def __init__(self, handler: SyncHandler[Any], settings: PluginSettings) -> None:
|
|
212
|
+
self._handler = handler
|
|
213
|
+
self._settings = settings
|
|
214
|
+
self._task: asyncio.Task[None] | None = None
|
|
215
|
+
self.state = SyncState()
|
|
216
|
+
|
|
217
|
+
def start(self) -> SyncState:
|
|
218
|
+
"""Start a sync unless one runs; the state either way."""
|
|
219
|
+
if not self.state.in_progress:
|
|
220
|
+
self.state.in_progress = True
|
|
221
|
+
self.state.last_sync_at = datetime.now(UTC)
|
|
222
|
+
self.state.last_error = None
|
|
223
|
+
self._task = asyncio.create_task(self._run())
|
|
224
|
+
return self.state.model_copy()
|
|
225
|
+
|
|
226
|
+
async def _run(self) -> None:
|
|
227
|
+
try:
|
|
228
|
+
count = await _run(self._handler, self._settings)
|
|
229
|
+
if not isinstance(count, int):
|
|
230
|
+
raise TypeError(
|
|
231
|
+
f"the sync handler returned {type(count).__name__}, not int"
|
|
232
|
+
)
|
|
233
|
+
self.state.last_sync_count = count
|
|
234
|
+
except Exception as error:
|
|
235
|
+
self.state.last_error = type(error).__name__
|
|
236
|
+
_log_failure("sync", error)
|
|
237
|
+
finally:
|
|
238
|
+
self.state.last_finished_at = datetime.now(UTC)
|
|
239
|
+
self.state.in_progress = False
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _token_check(token: SecretStr) -> Callable[[Request], Awaitable[None]]:
|
|
243
|
+
expected = token.get_secret_value().encode()
|
|
244
|
+
|
|
245
|
+
async def check_token(request: Request) -> None:
|
|
246
|
+
if not _is_bearer(request.headers.get("authorization"), expected):
|
|
247
|
+
raise HTTPException(
|
|
248
|
+
status_code=401,
|
|
249
|
+
detail="unauthorized",
|
|
250
|
+
headers={"WWW-Authenticate": "Bearer"},
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
return check_token
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _is_bearer(header: str | None, token: bytes) -> bool:
|
|
257
|
+
"""Whether ``header`` is ``Bearer <token>``; an empty token matches nothing."""
|
|
258
|
+
if not token or header is None:
|
|
259
|
+
return False
|
|
260
|
+
scheme, _, presented = header.partition(" ")
|
|
261
|
+
return scheme.lower() == "bearer" and hmac.compare_digest(presented.encode(), token)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
async def _parse(request: Request, model: type[ModelT]) -> ModelT:
|
|
265
|
+
"""The request body as ``model``; a 422 names the fields refused, never the values sent.
|
|
266
|
+
|
|
267
|
+
The 422 is raised after the handler, so the validation error, which holds
|
|
268
|
+
the request's content, is not attached as its context.
|
|
269
|
+
"""
|
|
270
|
+
try:
|
|
271
|
+
return model.model_validate_json(await request.body())
|
|
272
|
+
except ValidationError as error:
|
|
273
|
+
detail = error.errors(
|
|
274
|
+
include_url=False, include_context=False, include_input=False
|
|
275
|
+
)
|
|
276
|
+
raise HTTPException(status_code=422, detail=detail)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
async def _run(handler: Callable[..., Any], *args: Any) -> Any:
|
|
280
|
+
"""Await an async handler; run any other in a worker thread, in this context.
|
|
281
|
+
|
|
282
|
+
A handler is async when it is an ``async def`` function, or an object whose
|
|
283
|
+
``__call__`` is one, as FastAPI tells them apart.
|
|
284
|
+
"""
|
|
285
|
+
if inspect.iscoroutinefunction(handler) or inspect.iscoroutinefunction(
|
|
286
|
+
getattr(handler, "__call__", None)
|
|
287
|
+
):
|
|
288
|
+
return await handler(*args)
|
|
289
|
+
return await run_in_threadpool(handler, *args)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
@contextmanager
|
|
293
|
+
def _correlated(request: Request) -> Iterator[None]:
|
|
294
|
+
"""Make the call's ``X-Correlation-ID`` the one the clients send."""
|
|
295
|
+
reset = correlation_id.set(request.headers.get("x-correlation-id"))
|
|
296
|
+
try:
|
|
297
|
+
yield
|
|
298
|
+
finally:
|
|
299
|
+
correlation_id.reset(reset)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _log_failure(what: str, error: Exception) -> None:
|
|
303
|
+
"""Log a handler's failure by its type and place only: its message, or the
|
|
304
|
+
locations of a validation error, may quote the email, or a directory URL
|
|
305
|
+
with its credentials."""
|
|
306
|
+
frame = traceback.extract_tb(error.__traceback__)[-1]
|
|
307
|
+
logger.error(
|
|
308
|
+
"%s failed: %s at %s:%s in %s",
|
|
309
|
+
what,
|
|
310
|
+
type(error).__name__,
|
|
311
|
+
frame.filename,
|
|
312
|
+
frame.lineno,
|
|
313
|
+
frame.name,
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _log_invalid_answer(what: str, error: ValidationError) -> None:
|
|
318
|
+
"""Log a handler's answer that does not validate by the fields refused and
|
|
319
|
+
why (``loc`` and ``type``), never their values. The locations are the
|
|
320
|
+
answer model's own fields, so they hold none of the email."""
|
|
321
|
+
fields = ", ".join(
|
|
322
|
+
f"{'.'.join(str(part) for part in detail['loc'])} ({detail['type']})"
|
|
323
|
+
for detail in error.errors(
|
|
324
|
+
include_url=False, include_context=False, include_input=False
|
|
325
|
+
)
|
|
326
|
+
)
|
|
327
|
+
logger.error(
|
|
328
|
+
"%s failed: the handler's answer is not a valid %s; invalid: %s",
|
|
329
|
+
what,
|
|
330
|
+
error.title,
|
|
331
|
+
fields,
|
|
332
|
+
)
|