hedit 0.7.9.dev4__tar.gz → 0.7.10a2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hedit-0.7.9.dev4/hedit.egg-info → hedit-0.7.10a2}/PKG-INFO +1 -1
- {hedit-0.7.9.dev4 → hedit-0.7.10a2/hedit.egg-info}/PKG-INFO +1 -1
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/hedit.egg-info/SOURCES.txt +5 -2
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/hedit.egg-info/entry_points.txt +1 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/pyproject.toml +2 -1
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/agents/validation_agent.py +25 -21
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/agents/workflow.py +60 -54
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/api/main.py +102 -3
- hedit-0.7.10a2/src/cli/commands/lsp.py +186 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/cli/local_executor.py +83 -11
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/cli/main.py +2 -0
- hedit-0.7.10a2/src/lsp/__init__.py +13 -0
- hedit-0.7.10a2/src/lsp/client.py +358 -0
- hedit-0.7.10a2/src/lsp/daemon.py +405 -0
- hedit-0.7.10a2/src/lsp/protocol.py +69 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/utils/openrouter_llm.py +14 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/validation/__init__.py +4 -6
- hedit-0.7.10a2/src/validation/hed_lsp.py +53 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/version.py +2 -2
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_openrouter_llm.py +34 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_validation_agent.py +1 -1
- hedit-0.7.9.dev4/src/validation/hed_lsp.py +0 -378
- hedit-0.7.9.dev4/tests/test_hed_lsp.py +0 -376
- hedit-0.7.9.dev4/tests/test_keyword_extraction.py +0 -368
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/LICENSE +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/PKG_README.md +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/README.md +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/hedit.egg-info/dependency_links.txt +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/hedit.egg-info/requires.txt +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/hedit.egg-info/top_level.txt +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/setup.cfg +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/__init__.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/agents/__init__.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/agents/annotation_agent.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/agents/assessment_agent.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/agents/evaluation_agent.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/agents/feedback_summarizer.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/agents/feedback_triage_agent.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/agents/state.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/agents/vision_agent.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/api/__init__.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/api/models.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/api/security.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/cli/__init__.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/cli/api_executor.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/cli/client.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/cli/commands/__init__.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/cli/config.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/cli/executor.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/cli/output.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/data/__init__.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/data/hed-docs/02_Terminology.md +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/data/hed-docs/HedAnnotationSemantics.md +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/data/hed-docs/manifest.json +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/scripts/__init__.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/scripts/process_feedback.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/telemetry/__init__.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/telemetry/collector.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/telemetry/schema.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/telemetry/storage.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/utils/__init__.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/utils/error_remediation.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/utils/github_client.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/utils/hed_comprehensive_guide.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/utils/hed_docs_loader.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/utils/image_processing.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/utils/json_schema_loader.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/utils/litellm_llm.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/utils/schema_loader.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/src/validation/hed_validator.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_annotation_agent.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_api_endpoints.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_cli_client.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_cli_config.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_cli_integration.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_cli_main.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_comprehensive_guide.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_error_remediation.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_feedback_integration.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_feedback_triage.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_fetch_hed_docs.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_github_client.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_hed_docs_loader.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_integration_openrouter.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_json_schema_loader.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_litellm_llm.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_no_extend_propagation.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_schema_loader.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_security.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_state.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_telemetry.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_validation.py +0 -0
- {hedit-0.7.9.dev4 → hedit-0.7.10a2}/tests/test_version.py +0 -0
|
@@ -33,10 +33,15 @@ src/cli/local_executor.py
|
|
|
33
33
|
src/cli/main.py
|
|
34
34
|
src/cli/output.py
|
|
35
35
|
src/cli/commands/__init__.py
|
|
36
|
+
src/cli/commands/lsp.py
|
|
36
37
|
src/data/__init__.py
|
|
37
38
|
src/data/hed-docs/02_Terminology.md
|
|
38
39
|
src/data/hed-docs/HedAnnotationSemantics.md
|
|
39
40
|
src/data/hed-docs/manifest.json
|
|
41
|
+
src/lsp/__init__.py
|
|
42
|
+
src/lsp/client.py
|
|
43
|
+
src/lsp/daemon.py
|
|
44
|
+
src/lsp/protocol.py
|
|
40
45
|
src/scripts/__init__.py
|
|
41
46
|
src/scripts/process_feedback.py
|
|
42
47
|
src/telemetry/__init__.py
|
|
@@ -69,10 +74,8 @@ tests/test_feedback_triage.py
|
|
|
69
74
|
tests/test_fetch_hed_docs.py
|
|
70
75
|
tests/test_github_client.py
|
|
71
76
|
tests/test_hed_docs_loader.py
|
|
72
|
-
tests/test_hed_lsp.py
|
|
73
77
|
tests/test_integration_openrouter.py
|
|
74
78
|
tests/test_json_schema_loader.py
|
|
75
|
-
tests/test_keyword_extraction.py
|
|
76
79
|
tests/test_litellm_llm.py
|
|
77
80
|
tests/test_no_extend_propagation.py
|
|
78
81
|
tests/test_openrouter_llm.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "hedit"
|
|
7
|
-
version = "0.7.
|
|
7
|
+
version = "0.7.10a2"
|
|
8
8
|
description = "Multi-agent system for HED annotation generation and validation"
|
|
9
9
|
readme = "PKG_README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -39,6 +39,7 @@ dependencies = [
|
|
|
39
39
|
|
|
40
40
|
[project.scripts]
|
|
41
41
|
hedit = "src.cli.main:cli"
|
|
42
|
+
hedit-lspd = "src.lsp.daemon:main"
|
|
42
43
|
|
|
43
44
|
[project.urls]
|
|
44
45
|
Homepage = "https://annotation.garden/hedit"
|
|
@@ -12,9 +12,9 @@ import re
|
|
|
12
12
|
from pathlib import Path
|
|
13
13
|
|
|
14
14
|
from src.agents.state import HedAnnotationState
|
|
15
|
+
from src.lsp import HedLspClient
|
|
15
16
|
from src.utils.error_remediation import get_remediator
|
|
16
17
|
from src.utils.schema_loader import HedSchemaLoader
|
|
17
|
-
from src.validation.hed_lsp import is_hed_lsp_available, suggest_tags_for_keywords
|
|
18
18
|
from src.validation.hed_validator import (
|
|
19
19
|
HedJavaScriptValidator,
|
|
20
20
|
HedPythonValidator,
|
|
@@ -81,7 +81,7 @@ class ValidationAgent:
|
|
|
81
81
|
use_javascript: bool = True,
|
|
82
82
|
validator_path: Path | None = None,
|
|
83
83
|
tests_json_path: Path | str | None = None,
|
|
84
|
-
|
|
84
|
+
lsp_client: HedLspClient | None = None,
|
|
85
85
|
) -> None:
|
|
86
86
|
"""Initialize the validation agent.
|
|
87
87
|
|
|
@@ -90,13 +90,14 @@ class ValidationAgent:
|
|
|
90
90
|
use_javascript: Whether to use JavaScript validator (more detailed)
|
|
91
91
|
validator_path: Path to hed-javascript repository (required if use_javascript=True)
|
|
92
92
|
tests_json_path: Optional path to javascriptTests.json for error remediation
|
|
93
|
-
|
|
93
|
+
lsp_client: Pre-built HedLspClient for tag-replacement suggestions
|
|
94
|
+
when validation finds invalid tags. None disables suggestions.
|
|
94
95
|
"""
|
|
95
96
|
self.schema_loader = schema_loader
|
|
96
97
|
self.use_javascript = use_javascript
|
|
97
98
|
self.validator_path = validator_path
|
|
98
99
|
self.error_remediator = get_remediator(tests_json_path)
|
|
99
|
-
self.
|
|
100
|
+
self.lsp_client = lsp_client
|
|
100
101
|
|
|
101
102
|
# Validator is lazily initialized on first use via _get_or_create_validator
|
|
102
103
|
self._validator: HedJavaScriptValidator | HedPythonValidator | None = None
|
|
@@ -134,34 +135,37 @@ class ValidationAgent:
|
|
|
134
135
|
|
|
135
136
|
return problematic_tags
|
|
136
137
|
|
|
137
|
-
def _get_tag_suggestions(
|
|
138
|
-
|
|
139
|
-
) -> dict[str, list[str]]:
|
|
140
|
-
"""Get suggested valid tags for problematic tags using hed-lsp.
|
|
138
|
+
async def _get_tag_suggestions(self, problematic_tags: list[str]) -> dict[str, list[str]]:
|
|
139
|
+
"""Get suggested valid tags for problematic tags via persistent LSP.
|
|
141
140
|
|
|
142
141
|
Args:
|
|
143
142
|
problematic_tags: List of problematic tag names
|
|
144
|
-
schema_version: HED schema version
|
|
145
143
|
|
|
146
144
|
Returns:
|
|
147
|
-
Dictionary mapping problematic
|
|
145
|
+
Dictionary mapping each problematic tag to a list of suggested
|
|
146
|
+
valid alternatives. Empty if no LSP client is configured or the
|
|
147
|
+
server returned no matches.
|
|
148
148
|
"""
|
|
149
|
-
if
|
|
149
|
+
if self.lsp_client is None or not problematic_tags:
|
|
150
150
|
return {}
|
|
151
151
|
|
|
152
152
|
try:
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
except (RuntimeError, OSError) as e:
|
|
153
|
+
result = await self.lsp_client.suggest(*problematic_tags)
|
|
154
|
+
except Exception as exc:
|
|
155
|
+
# `suggest()` already converts transport errors to a failure
|
|
156
|
+
# result; anything that lands here is a programming error and
|
|
157
|
+
# deserves a traceback.
|
|
159
158
|
logger.warning(
|
|
160
|
-
"
|
|
159
|
+
"hed-lsp suggest call failed for tags %s: %s",
|
|
161
160
|
problematic_tags,
|
|
162
|
-
|
|
161
|
+
exc,
|
|
162
|
+
exc_info=True,
|
|
163
163
|
)
|
|
164
164
|
return {}
|
|
165
|
+
if not result.success:
|
|
166
|
+
logger.debug("hed-lsp suggest returned failure: %s", result.error)
|
|
167
|
+
return {}
|
|
168
|
+
return result.raw
|
|
165
169
|
|
|
166
170
|
async def validate(self, state: HedAnnotationState) -> dict:
|
|
167
171
|
"""Validate the current HED annotation.
|
|
@@ -202,10 +206,10 @@ class ValidationAgent:
|
|
|
202
206
|
|
|
203
207
|
# Extract problematic tags and get suggestions from hed-lsp
|
|
204
208
|
tag_suggestions: dict[str, list[str]] = {}
|
|
205
|
-
if not result.is_valid and self.
|
|
209
|
+
if not result.is_valid and self.lsp_client is not None:
|
|
206
210
|
problematic_tags = self._extract_problematic_tags(result.errors, result.warnings)
|
|
207
211
|
if problematic_tags:
|
|
208
|
-
tag_suggestions = self._get_tag_suggestions(problematic_tags
|
|
212
|
+
tag_suggestions = await self._get_tag_suggestions(problematic_tags)
|
|
209
213
|
|
|
210
214
|
# Determine validation status
|
|
211
215
|
validation_attempts = state["validation_attempts"] + 1
|
|
@@ -4,7 +4,6 @@ This module defines the multi-agent workflow that orchestrates
|
|
|
4
4
|
annotation, validation, evaluation, and assessment.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
import asyncio
|
|
8
7
|
import logging
|
|
9
8
|
import time
|
|
10
9
|
from pathlib import Path
|
|
@@ -19,9 +18,9 @@ from src.agents.evaluation_agent import EvaluationAgent
|
|
|
19
18
|
from src.agents.feedback_summarizer import FeedbackSummarizer
|
|
20
19
|
from src.agents.state import HedAnnotationState
|
|
21
20
|
from src.agents.validation_agent import ValidationAgent
|
|
21
|
+
from src.lsp import HedLspClient
|
|
22
22
|
from src.utils import extract_text_content
|
|
23
23
|
from src.utils.schema_loader import HedSchemaLoader
|
|
24
|
-
from src.validation.hed_lsp import HedLspClient, is_hed_lsp_available
|
|
25
24
|
|
|
26
25
|
logger = logging.getLogger(__name__)
|
|
27
26
|
|
|
@@ -47,10 +46,12 @@ class HedAnnotationWorkflow:
|
|
|
47
46
|
evaluation_llm: BaseChatModel | None = None,
|
|
48
47
|
assessment_llm: BaseChatModel | None = None,
|
|
49
48
|
feedback_llm: BaseChatModel | None = None,
|
|
49
|
+
keyword_llm: BaseChatModel | None = None,
|
|
50
50
|
schema_dir: Path | str | None = None,
|
|
51
51
|
validator_path: Path | None = None,
|
|
52
52
|
use_js_validator: bool = True,
|
|
53
53
|
enable_semantic_search: bool = True,
|
|
54
|
+
lsp_client: HedLspClient | None = None,
|
|
54
55
|
) -> None:
|
|
55
56
|
"""Initialize the workflow.
|
|
56
57
|
|
|
@@ -59,15 +60,28 @@ class HedAnnotationWorkflow:
|
|
|
59
60
|
evaluation_llm: Language model for evaluation agent (defaults to llm)
|
|
60
61
|
assessment_llm: Language model for assessment agent (defaults to llm)
|
|
61
62
|
feedback_llm: Language model for feedback summarization (defaults to llm)
|
|
63
|
+
keyword_llm: Lightweight LLM for keyword extraction inside
|
|
64
|
+
``_semantic_preprocess_node``. Defaults to ``llm`` (the
|
|
65
|
+
annotation LLM, typically a fast model like claude-haiku
|
|
66
|
+
with reasoning disabled). Previously this task piggy-backed
|
|
67
|
+
on ``feedback_llm`` which in prod is wired to the heavier
|
|
68
|
+
evaluation model — a five-keyword extraction does not
|
|
69
|
+
need a 35B-parameter model with extended reasoning.
|
|
62
70
|
schema_dir: Directory containing JSON schemas
|
|
63
71
|
validator_path: Path to hed-javascript for validation
|
|
64
72
|
use_js_validator: Whether to use JavaScript validator
|
|
65
|
-
enable_semantic_search: Whether to
|
|
73
|
+
enable_semantic_search: Whether to include the semantic_preprocess
|
|
74
|
+
node in the graph. The node is only useful when an
|
|
75
|
+
`lsp_client` is also provided.
|
|
76
|
+
lsp_client: Pre-built, already-initialized HedLspClient. The
|
|
77
|
+
caller (FastAPI lifespan, CLI executor) owns the client's
|
|
78
|
+
lifetime. None disables LSP-backed tag enrichment.
|
|
66
79
|
"""
|
|
67
80
|
# Store schema directory (None means use HED library to fetch from GitHub)
|
|
68
81
|
self.schema_dir = schema_dir
|
|
69
|
-
# Keyword extraction always runs; LSP enrichment requires
|
|
82
|
+
# Keyword extraction always runs; LSP enrichment requires an injected client
|
|
70
83
|
self.enable_semantic_search = enable_semantic_search
|
|
84
|
+
self.lsp_client: HedLspClient | None = lsp_client
|
|
71
85
|
|
|
72
86
|
# Initialize legacy schema loader for validation
|
|
73
87
|
self.schema_loader = HedSchemaLoader()
|
|
@@ -76,9 +90,12 @@ class HedAnnotationWorkflow:
|
|
|
76
90
|
eval_llm = evaluation_llm or llm
|
|
77
91
|
assess_llm = assessment_llm or llm
|
|
78
92
|
feed_llm = feedback_llm or llm
|
|
93
|
+
kw_llm = keyword_llm or llm
|
|
79
94
|
|
|
80
|
-
#
|
|
81
|
-
|
|
95
|
+
# Keyword extraction uses a dedicated lightweight model
|
|
96
|
+
# (defaults to the annotation LLM, which is typically fast +
|
|
97
|
+
# reasoning-disabled for this short structured task).
|
|
98
|
+
self.keyword_llm = kw_llm
|
|
82
99
|
|
|
83
100
|
# Initialize agents with JSON schema support and per-agent LLMs
|
|
84
101
|
self.annotation_agent = AnnotationAgent(llm, schema_dir=self.schema_dir)
|
|
@@ -86,22 +103,17 @@ class HedAnnotationWorkflow:
|
|
|
86
103
|
self.schema_loader,
|
|
87
104
|
use_javascript=use_js_validator,
|
|
88
105
|
validator_path=validator_path,
|
|
106
|
+
lsp_client=self.lsp_client,
|
|
89
107
|
)
|
|
90
108
|
self.evaluation_agent = EvaluationAgent(eval_llm, schema_dir=self.schema_dir)
|
|
91
109
|
self.assessment_agent = AssessmentAgent(assess_llm, schema_dir=self.schema_dir)
|
|
92
110
|
self.feedback_summarizer = FeedbackSummarizer(feed_llm)
|
|
93
111
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
if self.enable_semantic_search and is_hed_lsp_available():
|
|
97
|
-
try:
|
|
98
|
-
self.hed_lsp_client = HedLspClient()
|
|
99
|
-
logger.info("[WORKFLOW] hed-lsp CLI available for semantic tag suggestions")
|
|
100
|
-
except RuntimeError as e:
|
|
101
|
-
logger.warning(f"[WORKFLOW] hed-lsp CLI not available: {e}")
|
|
112
|
+
if self.enable_semantic_search and self.lsp_client is not None:
|
|
113
|
+
logger.info("[WORKFLOW] hed-lsp client connected for semantic tag suggestions")
|
|
102
114
|
elif self.enable_semantic_search:
|
|
103
115
|
logger.info(
|
|
104
|
-
"[WORKFLOW]
|
|
116
|
+
"[WORKFLOW] semantic_preprocess enabled but no lsp_client provided; "
|
|
105
117
|
"keyword extraction will run without LSP enrichment"
|
|
106
118
|
)
|
|
107
119
|
|
|
@@ -195,7 +207,7 @@ class HedAnnotationWorkflow:
|
|
|
195
207
|
)
|
|
196
208
|
|
|
197
209
|
try:
|
|
198
|
-
response = await self.
|
|
210
|
+
response = await self.keyword_llm.ainvoke(
|
|
199
211
|
[
|
|
200
212
|
SystemMessage(content=system_prompt),
|
|
201
213
|
HumanMessage(content=f"Description: {description}"),
|
|
@@ -236,52 +248,46 @@ class HedAnnotationWorkflow:
|
|
|
236
248
|
# Step 1: Extract keywords from the description using LLM
|
|
237
249
|
keywords = await self._extract_keywords(state["input_description"])
|
|
238
250
|
|
|
239
|
-
# Step 2:
|
|
251
|
+
# Step 2: One batched hed/suggest call over the persistent LSP
|
|
252
|
+
# connection. The previous per-keyword loop spawned the hed-suggest
|
|
253
|
+
# CLI once per keyword and dominated request latency (~6 s/spawn).
|
|
240
254
|
semantic_hints: list[dict] = []
|
|
241
255
|
|
|
242
|
-
if keywords and self.
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
256
|
+
if keywords and self.lsp_client is not None:
|
|
257
|
+
try:
|
|
258
|
+
result = await self.lsp_client.suggest(*keywords)
|
|
259
|
+
except Exception as e:
|
|
260
|
+
# `suggest()` already swallows transport errors and returns a
|
|
261
|
+
# failure result; anything that reaches here is unexpected
|
|
262
|
+
# and deserves a full traceback in logs.
|
|
263
|
+
logger.warning("[WORKFLOW] hed-lsp suggest failed: %s", e, exc_info=True)
|
|
264
|
+
result = None
|
|
265
|
+
|
|
266
|
+
if result is not None and result.success:
|
|
267
|
+
seen_tags: dict[str, dict] = {}
|
|
268
|
+
for keyword, tags in result.raw.items():
|
|
269
|
+
for idx, tag in enumerate(tags):
|
|
270
|
+
# Higher score for earlier (more relevant) hits.
|
|
271
|
+
score = max(0.0, 1.0 - idx * 0.05)
|
|
272
|
+
existing = seen_tags.get(tag)
|
|
273
|
+
if existing is None or score > existing["score"]:
|
|
274
|
+
seen_tags[tag] = {
|
|
275
|
+
"tag": tag,
|
|
255
276
|
"keyword": keyword,
|
|
256
|
-
"score":
|
|
277
|
+
"score": score,
|
|
257
278
|
"source": "hed-lsp",
|
|
258
279
|
}
|
|
259
|
-
)
|
|
260
|
-
else:
|
|
261
|
-
logger.debug(
|
|
262
|
-
"[WORKFLOW] hed-lsp suggestion failed for '%s': %s",
|
|
263
|
-
keyword,
|
|
264
|
-
result.error,
|
|
265
|
-
)
|
|
266
|
-
|
|
267
|
-
# Deduplicate by tag, keeping highest score
|
|
268
|
-
if semantic_hints:
|
|
269
|
-
seen_tags: dict[str, dict] = {}
|
|
270
|
-
for hint in semantic_hints:
|
|
271
|
-
tag = hint["tag"]
|
|
272
|
-
if tag not in seen_tags or hint["score"] > seen_tags[tag]["score"]:
|
|
273
|
-
seen_tags[tag] = hint
|
|
274
280
|
semantic_hints = sorted(seen_tags.values(), key=lambda h: h["score"], reverse=True)
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
+
logger.info(
|
|
282
|
+
"[WORKFLOW] hed-lsp suggested %d unique tags from %d keywords",
|
|
283
|
+
len(semantic_hints),
|
|
284
|
+
len(keywords),
|
|
285
|
+
)
|
|
286
|
+
elif result is not None:
|
|
287
|
+
logger.debug("[WORKFLOW] hed-lsp suggest returned failure: %s", result.error)
|
|
281
288
|
elif keywords:
|
|
282
|
-
# LSP not available; still store keywords for the annotation agent
|
|
283
289
|
logger.info(
|
|
284
|
-
"[WORKFLOW]
|
|
290
|
+
"[WORKFLOW] no lsp_client; storing %d extracted keywords without enrichment",
|
|
285
291
|
len(keywords),
|
|
286
292
|
)
|
|
287
293
|
|
|
@@ -35,6 +35,7 @@ from src.api.models import (
|
|
|
35
35
|
ValidationResponse,
|
|
36
36
|
)
|
|
37
37
|
from src.api.security import api_key_auth, audit_logger
|
|
38
|
+
from src.lsp import HedLspClient
|
|
38
39
|
from src.telemetry import LocalFileStorage, TelemetryCollector, TelemetryEvent
|
|
39
40
|
from src.utils.openrouter_llm import create_openrouter_llm, get_model_name
|
|
40
41
|
from src.utils.schema_loader import HedSchemaLoader
|
|
@@ -48,6 +49,11 @@ workflow: HedAnnotationWorkflow | None = None
|
|
|
48
49
|
vision_agent: VisionAgent | None = None
|
|
49
50
|
schema_loader: HedSchemaLoader | None = None
|
|
50
51
|
|
|
52
|
+
# Persistent LSP client; lifetime = process lifetime. Spawned in lifespan
|
|
53
|
+
# so every request reuses one warm Node child instead of cold-booting
|
|
54
|
+
# hed-suggest per keyword.
|
|
55
|
+
lsp_client: HedLspClient | None = None
|
|
56
|
+
|
|
51
57
|
# Telemetry collector (initialized in lifespan)
|
|
52
58
|
telemetry_collector: TelemetryCollector | None = None
|
|
53
59
|
|
|
@@ -78,6 +84,25 @@ def _derive_user_id(token: str) -> str:
|
|
|
78
84
|
return derived.hex()
|
|
79
85
|
|
|
80
86
|
|
|
87
|
+
# Sentinel used by the workflow factories to mean "fall back to the
|
|
88
|
+
# server-wide lsp_client global". Per-request callers can pass an
|
|
89
|
+
# explicit None to opt out, or another HedLspClient instance to override.
|
|
90
|
+
_USE_GLOBAL_LSP: HedLspClient = object() # type: ignore[assignment]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _resolve_lsp_client(
|
|
94
|
+
explicit: HedLspClient | None,
|
|
95
|
+
) -> HedLspClient | None:
|
|
96
|
+
"""Pick the per-request LSP client.
|
|
97
|
+
|
|
98
|
+
Explicit values (including ``None``) win. The sentinel
|
|
99
|
+
``_USE_GLOBAL_LSP`` means "use the lifespan-managed global".
|
|
100
|
+
"""
|
|
101
|
+
if explicit is _USE_GLOBAL_LSP:
|
|
102
|
+
return lsp_client
|
|
103
|
+
return explicit
|
|
104
|
+
|
|
105
|
+
|
|
81
106
|
def create_openrouter_workflow(
|
|
82
107
|
api_key: str,
|
|
83
108
|
annotation_model: str | None = None,
|
|
@@ -89,6 +114,7 @@ def create_openrouter_workflow(
|
|
|
89
114
|
schema_dir: str | Path | None = None,
|
|
90
115
|
validator_path: str | Path | None = None,
|
|
91
116
|
use_js_validator: bool = True,
|
|
117
|
+
lsp_client: HedLspClient | None = _USE_GLOBAL_LSP,
|
|
92
118
|
) -> HedAnnotationWorkflow:
|
|
93
119
|
"""Create a workflow with OpenRouter LLMs.
|
|
94
120
|
|
|
@@ -133,7 +159,10 @@ def create_openrouter_workflow(
|
|
|
133
159
|
|
|
134
160
|
actual_eval_provider = eval_provider or default_eval_provider or None
|
|
135
161
|
|
|
136
|
-
# Create LLMs
|
|
162
|
+
# Create LLMs.
|
|
163
|
+
# The annotation LLM keeps reasoning enabled — that's the model
|
|
164
|
+
# doing the actual HED tag synthesis where extended thinking
|
|
165
|
+
# measurably improves first-attempt quality.
|
|
137
166
|
annotation_llm = create_openrouter_llm(
|
|
138
167
|
model=actual_annotation_model,
|
|
139
168
|
api_key=api_key,
|
|
@@ -141,12 +170,16 @@ def create_openrouter_workflow(
|
|
|
141
170
|
provider=actual_annotation_provider,
|
|
142
171
|
user_id=actual_user_id,
|
|
143
172
|
)
|
|
173
|
+
# Evaluation / assessment / feedback / keyword extraction are short
|
|
174
|
+
# structured tasks; reasoning adds 5-10 s per call without
|
|
175
|
+
# measurable quality benefit. See #150.
|
|
144
176
|
evaluation_llm = create_openrouter_llm(
|
|
145
177
|
model=actual_eval_model,
|
|
146
178
|
api_key=api_key,
|
|
147
179
|
temperature=actual_temperature,
|
|
148
180
|
provider=actual_eval_provider,
|
|
149
181
|
user_id=actual_user_id,
|
|
182
|
+
disable_reasoning=True,
|
|
150
183
|
)
|
|
151
184
|
assessment_llm = create_openrouter_llm(
|
|
152
185
|
model=actual_eval_model,
|
|
@@ -154,6 +187,7 @@ def create_openrouter_workflow(
|
|
|
154
187
|
temperature=actual_temperature,
|
|
155
188
|
provider=actual_eval_provider,
|
|
156
189
|
user_id=actual_user_id,
|
|
190
|
+
disable_reasoning=True,
|
|
157
191
|
)
|
|
158
192
|
feedback_llm = create_openrouter_llm(
|
|
159
193
|
model=actual_eval_model,
|
|
@@ -161,20 +195,37 @@ def create_openrouter_workflow(
|
|
|
161
195
|
temperature=actual_temperature,
|
|
162
196
|
provider=actual_eval_provider,
|
|
163
197
|
user_id=actual_user_id,
|
|
198
|
+
disable_reasoning=True,
|
|
199
|
+
)
|
|
200
|
+
# Keyword extraction (#148): use the fast annotation model with
|
|
201
|
+
# reasoning explicitly disabled and a small token cap. The
|
|
202
|
+
# task is "list 5-10 keywords"; the heavier eval model used here
|
|
203
|
+
# previously cost ~10 s per call.
|
|
204
|
+
keyword_llm = create_openrouter_llm(
|
|
205
|
+
model=actual_annotation_model,
|
|
206
|
+
api_key=api_key,
|
|
207
|
+
temperature=actual_temperature,
|
|
208
|
+
provider=actual_annotation_provider,
|
|
209
|
+
user_id=actual_user_id,
|
|
210
|
+
max_tokens=200,
|
|
211
|
+
disable_reasoning=True,
|
|
164
212
|
)
|
|
165
213
|
|
|
166
214
|
# Create and return workflow
|
|
167
|
-
# Only use JS validator if validator_path is available
|
|
168
|
-
#
|
|
215
|
+
# Only use JS validator if validator_path is available.
|
|
216
|
+
# The LSP client (when provided) backs the persistent hed-lsp connection
|
|
217
|
+
# used for both semantic preprocessing and tag-replacement suggestions.
|
|
169
218
|
actual_use_js = use_js_validator and validator_path is not None
|
|
170
219
|
return HedAnnotationWorkflow(
|
|
171
220
|
llm=annotation_llm,
|
|
172
221
|
evaluation_llm=evaluation_llm,
|
|
173
222
|
assessment_llm=assessment_llm,
|
|
174
223
|
feedback_llm=feedback_llm,
|
|
224
|
+
keyword_llm=keyword_llm,
|
|
175
225
|
schema_dir=Path(schema_dir) if schema_dir else None,
|
|
176
226
|
validator_path=Path(validator_path) if validator_path else None,
|
|
177
227
|
use_js_validator=actual_use_js,
|
|
228
|
+
lsp_client=_resolve_lsp_client(lsp_client),
|
|
178
229
|
)
|
|
179
230
|
|
|
180
231
|
|
|
@@ -186,6 +237,7 @@ def create_byok_workflow(
|
|
|
186
237
|
eval_provider: str | None = None,
|
|
187
238
|
temperature: float | None = None,
|
|
188
239
|
user_id_override: str | None = None,
|
|
240
|
+
lsp_client: HedLspClient | None = _USE_GLOBAL_LSP,
|
|
189
241
|
) -> HedAnnotationWorkflow:
|
|
190
242
|
"""Create a workflow for BYOK mode using the user's OpenRouter key.
|
|
191
243
|
|
|
@@ -217,6 +269,7 @@ def create_byok_workflow(
|
|
|
217
269
|
schema_dir=_byok_config.get("schema_dir"),
|
|
218
270
|
validator_path=_byok_config.get("validator_path"),
|
|
219
271
|
use_js_validator=_byok_config.get("use_js_validator", True),
|
|
272
|
+
lsp_client=lsp_client,
|
|
220
273
|
)
|
|
221
274
|
|
|
222
275
|
|
|
@@ -342,6 +395,44 @@ async def lifespan(app: FastAPI):
|
|
|
342
395
|
print(f"Schema directory: {schema_dir or 'GitHub (dynamic fetch)'}")
|
|
343
396
|
print(f"Validator path: {validator_path or 'None (using Python validator)'}")
|
|
344
397
|
|
|
398
|
+
# Spawn the persistent hed-lsp child once for the lifetime of the
|
|
399
|
+
# process. Per-request workflows reuse this single warm connection
|
|
400
|
+
# instead of cold-booting hed-suggest per keyword. Set HED_LSP_DISABLE=1
|
|
401
|
+
# to fall back to keyword-only preprocessing (no LSP enrichment).
|
|
402
|
+
# Failures here are logged via the structured logger (not print) so
|
|
403
|
+
# operators can detect a silently-degraded server from log
|
|
404
|
+
# aggregation.
|
|
405
|
+
global lsp_client
|
|
406
|
+
lsp_logger = logging.getLogger("hedit.lsp")
|
|
407
|
+
if os.getenv("HED_LSP_DISABLE", "").lower() in ("1", "true", "yes"):
|
|
408
|
+
lsp_logger.info("LSP client disabled via HED_LSP_DISABLE")
|
|
409
|
+
lsp_client = None
|
|
410
|
+
else:
|
|
411
|
+
server_js_env = os.getenv("HED_LSP_SERVER_JS")
|
|
412
|
+
default_server_js = Path("/app/hed-lsp/server/out/server.js")
|
|
413
|
+
server_js_path = Path(server_js_env) if server_js_env else default_server_js
|
|
414
|
+
if not server_js_path.exists():
|
|
415
|
+
lsp_logger.error(
|
|
416
|
+
"LSP server.js not found at %s; set HED_LSP_SERVER_JS or install "
|
|
417
|
+
"hed-lsp. Continuing without LSP enrichment.",
|
|
418
|
+
server_js_path,
|
|
419
|
+
)
|
|
420
|
+
lsp_client = None
|
|
421
|
+
else:
|
|
422
|
+
try:
|
|
423
|
+
lsp_client = await HedLspClient.spawn_stdio(
|
|
424
|
+
server_js_path,
|
|
425
|
+
schema_version=os.getenv("HED_SCHEMA_VERSION", "8.4.0"),
|
|
426
|
+
)
|
|
427
|
+
lsp_logger.info("LSP client connected (server.js=%s)", server_js_path)
|
|
428
|
+
except (RuntimeError, OSError, TimeoutError) as exc:
|
|
429
|
+
lsp_logger.error(
|
|
430
|
+
"LSP client spawn failed; continuing without LSP enrichment. "
|
|
431
|
+
"Set HED_LSP_DISABLE=1 to suppress this error.",
|
|
432
|
+
exc_info=exc,
|
|
433
|
+
)
|
|
434
|
+
lsp_client = None
|
|
435
|
+
|
|
345
436
|
# Initialize workflow based on provider
|
|
346
437
|
if llm_provider == "openrouter":
|
|
347
438
|
# OpenRouter configuration - use unified workflow creation
|
|
@@ -364,6 +455,7 @@ async def lifespan(app: FastAPI):
|
|
|
364
455
|
schema_dir=schema_dir,
|
|
365
456
|
validator_path=validator_path if use_js_validator else None,
|
|
366
457
|
use_js_validator=use_js_validator,
|
|
458
|
+
lsp_client=lsp_client,
|
|
367
459
|
)
|
|
368
460
|
else:
|
|
369
461
|
# Ollama configuration (default)
|
|
@@ -384,6 +476,7 @@ async def lifespan(app: FastAPI):
|
|
|
384
476
|
schema_dir=Path(schema_dir) if schema_dir else None,
|
|
385
477
|
validator_path=Path(validator_path) if use_js_validator and validator_path else None,
|
|
386
478
|
use_js_validator=use_js_validator,
|
|
479
|
+
lsp_client=lsp_client,
|
|
387
480
|
)
|
|
388
481
|
|
|
389
482
|
# Set global schema_loader from workflow
|
|
@@ -428,6 +521,12 @@ async def lifespan(app: FastAPI):
|
|
|
428
521
|
|
|
429
522
|
# Shutdown
|
|
430
523
|
print("Shutting down HEDit...")
|
|
524
|
+
if lsp_client is not None:
|
|
525
|
+
try:
|
|
526
|
+
await lsp_client.shutdown()
|
|
527
|
+
lsp_logger.info("LSP client shut down cleanly")
|
|
528
|
+
except Exception as exc:
|
|
529
|
+
lsp_logger.error("LSP client shutdown error", exc_info=exc)
|
|
431
530
|
|
|
432
531
|
|
|
433
532
|
# Create FastAPI app
|