pleiad 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pleiad/__init__.py +0 -0
- pleiad/agent/__init__.py +0 -0
- pleiad/agent/agent.py +62 -0
- pleiad/agent/errors.py +21 -0
- pleiad/agent/policy.py +18 -0
- pleiad/agent/prompt.py +37 -0
- pleiad/capabilities/__init__.py +0 -0
- pleiad/capabilities/errors.py +51 -0
- pleiad/capabilities/search.py +193 -0
- pleiad/context/__init__.py +0 -0
- pleiad/context/budget.py +121 -0
- pleiad/context/compaction/__init__.py +0 -0
- pleiad/context/compaction/compactor.py +194 -0
- pleiad/context/compaction/models.py +26 -0
- pleiad/context/errors.py +21 -0
- pleiad/context/rendering.py +36 -0
- pleiad/context/selection/__init__.py +0 -0
- pleiad/context/selection/builder.py +43 -0
- pleiad/context/selection/models.py +24 -0
- pleiad/contracts.py +153 -0
- pleiad/errors/__init__.py +0 -0
- pleiad/errors/base.py +59 -0
- pleiad/events/__init__.py +0 -0
- pleiad/events/contracts.py +21 -0
- pleiad/events/graph.py +46 -0
- pleiad/events/message.py +62 -0
- pleiad/events/runtime.py +166 -0
- pleiad/events/supervisor.py +63 -0
- pleiad/logging/__init__.py +0 -0
- pleiad/logging/context.py +88 -0
- pleiad/logging/filters.py +23 -0
- pleiad/logging/rotation.py +44 -0
- pleiad/mcp/__init__.py +0 -0
- pleiad/mcp/contracts.py +33 -0
- pleiad/mcp/discovery.py +91 -0
- pleiad/mcp/errors.py +60 -0
- pleiad/mcp/tool.py +125 -0
- pleiad/memory/__init__.py +0 -0
- pleiad/memory/backends/__init__.py +0 -0
- pleiad/memory/backends/in_memory.py +139 -0
- pleiad/memory/backends/sqlite.py +296 -0
- pleiad/memory/errors.py +60 -0
- pleiad/memory/short_memory.py +195 -0
- pleiad/memory/turn.py +135 -0
- pleiad/messages/__init__.py +0 -0
- pleiad/messages/message.py +109 -0
- pleiad/models/__init__.py +0 -0
- pleiad/models/anthropic/__init__.py +0 -0
- pleiad/models/anthropic/client.py +145 -0
- pleiad/models/anthropic/request.py +84 -0
- pleiad/models/anthropic/response.py +335 -0
- pleiad/models/contracts.py +12 -0
- pleiad/models/errors.py +152 -0
- pleiad/models/openai/__init__.py +0 -0
- pleiad/models/openai/client.py +145 -0
- pleiad/models/openai/request.py +114 -0
- pleiad/models/openai/response.py +431 -0
- pleiad/models/providers/__init__.py +0 -0
- pleiad/models/providers/anthropic.py +129 -0
- pleiad/models/providers/deepseek.py +235 -0
- pleiad/models/providers/glm.py +263 -0
- pleiad/models/providers/kimi.py +253 -0
- pleiad/models/providers/minimax.py +235 -0
- pleiad/models/providers/openai.py +120 -0
- pleiad/models/providers/qwen.py +358 -0
- pleiad/models/result.py +135 -0
- pleiad/observability/__init__.py +0 -0
- pleiad/observability/attributes.py +66 -0
- pleiad/observability/config.py +31 -0
- pleiad/observability/errors.py +17 -0
- pleiad/observability/integrations/__init__.py +0 -0
- pleiad/observability/integrations/langfuse.py +163 -0
- pleiad/observability/metrics.py +68 -0
- pleiad/observability/privacy.py +191 -0
- pleiad/observability/tracing.py +168 -0
- pleiad/orchestration/__init__.py +0 -0
- pleiad/orchestration/errors.py +49 -0
- pleiad/orchestration/graph/__init__.py +0 -0
- pleiad/orchestration/graph/edge.py +59 -0
- pleiad/orchestration/graph/errors.py +41 -0
- pleiad/orchestration/graph/executor.py +496 -0
- pleiad/orchestration/graph/graph.py +82 -0
- pleiad/orchestration/graph/mapping.py +55 -0
- pleiad/orchestration/graph/node.py +86 -0
- pleiad/orchestration/graph/ports.py +12 -0
- pleiad/orchestration/graph/scheduler.py +20 -0
- pleiad/orchestration/graph/state.py +95 -0
- pleiad/orchestration/graph/validation.py +137 -0
- pleiad/orchestration/handoff/__init__.py +0 -0
- pleiad/orchestration/handoff/handoff.py +34 -0
- pleiad/orchestration/handoff/state.py +30 -0
- pleiad/orchestration/handoff/tools.py +11 -0
- pleiad/orchestration/supervisor/__init__.py +0 -0
- pleiad/orchestration/supervisor/state.py +31 -0
- pleiad/orchestration/supervisor/supervisor.py +480 -0
- pleiad/orchestration/supervisor/tools.py +214 -0
- pleiad/orchestration/supervisor/worker.py +69 -0
- pleiad/py.typed +2 -0
- pleiad/runtime/__init__.py +0 -0
- pleiad/runtime/config.py +23 -0
- pleiad/runtime/errors.py +25 -0
- pleiad/runtime/execution/__init__.py +0 -0
- pleiad/runtime/execution/control.py +37 -0
- pleiad/runtime/execution/loop.py +1075 -0
- pleiad/runtime/execution/model_input.py +495 -0
- pleiad/runtime/execution/runner.py +847 -0
- pleiad/runtime/execution/stream.py +68 -0
- pleiad/runtime/lifecycle/__init__.py +0 -0
- pleiad/runtime/lifecycle/result.py +38 -0
- pleiad/runtime/lifecycle/state.py +96 -0
- pleiad/runtime/policy.py +84 -0
- pleiad/sandbox/__init__.py +0 -0
- pleiad/sandbox/backends/__init__.py +0 -0
- pleiad/sandbox/backends/oci.py +485 -0
- pleiad/sandbox/backends/process.py +136 -0
- pleiad/sandbox/errors.py +44 -0
- pleiad/sandbox/models.py +120 -0
- pleiad/sandbox/policy.py +52 -0
- pleiad/sandbox/sandbox.py +18 -0
- pleiad/schemas.py +38 -0
- pleiad/skills/__init__.py +0 -0
- pleiad/skills/contracts.py +37 -0
- pleiad/skills/errors.py +25 -0
- pleiad/skills/loader.py +88 -0
- pleiad/skills/manifest.py +129 -0
- pleiad/skills/package.py +113 -0
- pleiad/tools/__init__.py +0 -0
- pleiad/tools/base.py +113 -0
- pleiad/tools/builtins/__init__.py +0 -0
- pleiad/tools/builtins/command.py +132 -0
- pleiad/tools/builtins/defaults.py +60 -0
- pleiad/tools/builtins/file_support.py +188 -0
- pleiad/tools/builtins/read_file.py +113 -0
- pleiad/tools/builtins/write_file.py +120 -0
- pleiad/tools/contracts.py +94 -0
- pleiad/tools/errors.py +211 -0
- pleiad/tools/interruption.py +37 -0
- pleiad/tools/manifest.py +70 -0
- pleiad/tools/registry.py +76 -0
- pleiad/tools/runtime.py +191 -0
- pleiad/version.py +3 -0
- pleiad-0.1.0.dist-info/METADATA +317 -0
- pleiad-0.1.0.dist-info/RECORD +146 -0
- pleiad-0.1.0.dist-info/WHEEL +5 -0
- pleiad-0.1.0.dist-info/licenses/LICENSE +21 -0
- pleiad-0.1.0.dist-info/top_level.txt +1 -0
pleiad/__init__.py
ADDED
|
File without changes
|
pleiad/agent/__init__.py
ADDED
|
File without changes
|
pleiad/agent/agent.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
|
|
3
|
+
from jsonschema import SchemaError
|
|
4
|
+
|
|
5
|
+
from pleiad.agent.errors import AgentConfigurationError
|
|
6
|
+
from pleiad.agent.policy import AgentPolicyHints
|
|
7
|
+
from pleiad.contracts import JsonSchema, SkillRef, ToolRef, freeze_json_object
|
|
8
|
+
from pleiad.models.anthropic.client import AnthropicModelClient
|
|
9
|
+
from pleiad.models.openai.client import OpenAIModelClient
|
|
10
|
+
from pleiad.schemas import validate_json_schema
|
|
11
|
+
from pleiad.tools.contracts import DEFAULT_TOOL_CODES, default_tool_refs
|
|
12
|
+
|
|
13
|
+
SUPPORTED_SCHEMA_DIALECT = "https://json-schema.org/draft/2020-12/schema"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True, slots=True)
|
|
17
|
+
class Agent:
|
|
18
|
+
"""Reusable Agent recipe containing only immutable declarations and refs."""
|
|
19
|
+
|
|
20
|
+
name: str
|
|
21
|
+
model: OpenAIModelClient | AnthropicModelClient
|
|
22
|
+
prompt: str
|
|
23
|
+
tools: tuple[ToolRef, ...] = ()
|
|
24
|
+
skills: tuple[SkillRef, ...] = ()
|
|
25
|
+
output_schema: JsonSchema | None = None
|
|
26
|
+
policy_hints: AgentPolicyHints = field(default_factory=AgentPolicyHints)
|
|
27
|
+
|
|
28
|
+
def __post_init__(self) -> None:
|
|
29
|
+
"""Validate the identity, schema, and capability references."""
|
|
30
|
+
if not self.name.strip():
|
|
31
|
+
raise AgentConfigurationError("Agent name must not be blank.")
|
|
32
|
+
if len(self.name) > 256:
|
|
33
|
+
raise AgentConfigurationError("Agent name must not exceed 256 characters.")
|
|
34
|
+
if not isinstance(self.model, OpenAIModelClient | AnthropicModelClient):
|
|
35
|
+
raise AgentConfigurationError("Agent model must be an OpenAIModelClient or AnthropicModelClient.")
|
|
36
|
+
if not self.prompt.strip():
|
|
37
|
+
raise AgentConfigurationError("Agent prompt must not be blank.")
|
|
38
|
+
explicit_tool_codes = tuple(ref.code for ref in self.tools)
|
|
39
|
+
if len(explicit_tool_codes) != len(set(explicit_tool_codes)):
|
|
40
|
+
raise AgentConfigurationError("Agent Tool refs must be unique.")
|
|
41
|
+
skill_codes = tuple(ref.code for ref in self.skills)
|
|
42
|
+
if len(skill_codes) != len(set(skill_codes)):
|
|
43
|
+
raise AgentConfigurationError("Agent Skill refs must be unique.")
|
|
44
|
+
if self.output_schema is not None:
|
|
45
|
+
validate_output_schema(self.output_schema)
|
|
46
|
+
object.__setattr__(self, "output_schema", freeze_json_object(self.output_schema))
|
|
47
|
+
default_codes = frozenset(DEFAULT_TOOL_CODES)
|
|
48
|
+
merged_tools = default_tool_refs() + tuple(ref for ref in self.tools if ref.code not in default_codes)
|
|
49
|
+
object.__setattr__(self, "tools", merged_tools)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def validate_output_schema(schema: JsonSchema) -> None:
|
|
53
|
+
"""Validate one local Draft 2020-12 JSON Schema without network resolution."""
|
|
54
|
+
dialect = schema.get("$schema")
|
|
55
|
+
if dialect is not None and dialect != SUPPORTED_SCHEMA_DIALECT:
|
|
56
|
+
raise AgentConfigurationError("Agent output_schema must use the Draft 2020-12 JSON Schema dialect.")
|
|
57
|
+
try:
|
|
58
|
+
validate_json_schema(schema)
|
|
59
|
+
except ValueError as error:
|
|
60
|
+
raise AgentConfigurationError(str(error)) from error
|
|
61
|
+
except SchemaError as error:
|
|
62
|
+
raise AgentConfigurationError("Agent output_schema is invalid.") from error
|
pleiad/agent/errors.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from pleiad.errors.base import BusinessError, SystemFault
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class AgentConfigurationError(SystemFault):
|
|
5
|
+
"""Report an invalid Agent definition or unavailable capability binding."""
|
|
6
|
+
|
|
7
|
+
def __init__(self, message: str = "The Agent configuration is invalid.") -> None:
|
|
8
|
+
"""Create an Agent configuration fault."""
|
|
9
|
+
super().__init__("agent.config.invalid", message)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class InvalidAgentOutputError(BusinessError):
|
|
13
|
+
"""Report a final model output that violates the Agent contract."""
|
|
14
|
+
|
|
15
|
+
def __init__(self, message: str = "The model output does not satisfy the Agent result contract.") -> None:
|
|
16
|
+
"""Create an invalid output error.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
message: Diagnostic description of the invalid output.
|
|
20
|
+
"""
|
|
21
|
+
super().__init__("agent.output.invalid", message)
|
pleiad/agent/policy.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from pydantic import ConfigDict, Field
|
|
2
|
+
|
|
3
|
+
from pleiad.contracts import CapabilityGrants, ContractModel
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class AgentPolicyHints(ContractModel):
|
|
7
|
+
"""Optional bounds requested by an Agent author.
|
|
8
|
+
|
|
9
|
+
Hints are never an authorization grant. The host intersects them with the
|
|
10
|
+
request and parent policy before a Run starts.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
model_config = ConfigDict(strict=True)
|
|
14
|
+
|
|
15
|
+
timeout_seconds: float | None = Field(default=None, gt=0, allow_inf_nan=False)
|
|
16
|
+
max_turns: int | None = Field(default=None, gt=0)
|
|
17
|
+
max_output_tokens: int | None = Field(default=None, gt=0)
|
|
18
|
+
grants: CapabilityGrants | None = None
|
pleiad/agent/prompt.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from pleiad.skills.contracts import Skill
|
|
2
|
+
from pleiad.tools.contracts import ToolDescriptor
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def build_prompt(
|
|
6
|
+
base_prompt: str,
|
|
7
|
+
*,
|
|
8
|
+
tools: tuple[ToolDescriptor, ...],
|
|
9
|
+
skills: tuple[Skill, ...],
|
|
10
|
+
) -> str:
|
|
11
|
+
"""Compose a prompt from the capabilities visible in the current Run."""
|
|
12
|
+
sections = [base_prompt.strip()]
|
|
13
|
+
tool_codes = frozenset(tool.code for tool in tools)
|
|
14
|
+
guidance: list[str] = []
|
|
15
|
+
if "read_file" in tool_codes:
|
|
16
|
+
guidance.append("- read_file: read bounded UTF-8 text from a relative workspace, Skill, or scratch path.")
|
|
17
|
+
if "write_file" in tool_codes:
|
|
18
|
+
guidance.append("- write_file: create or conditionally replace UTF-8 text at a relative workspace path.")
|
|
19
|
+
if "command" in tool_codes:
|
|
20
|
+
guidance.append("- command: run one command inside the configured Sandbox.")
|
|
21
|
+
if guidance:
|
|
22
|
+
sections.append("The following built-in tools are available:\n" + "\n".join(guidance))
|
|
23
|
+
if "read_file" in tool_codes:
|
|
24
|
+
sections.append(
|
|
25
|
+
"Use relative POSIX paths only. For a selected Skill, use read_file with "
|
|
26
|
+
'scope="skill", skill_code="<code>", and path="SKILL.md" to load its instructions.'
|
|
27
|
+
)
|
|
28
|
+
if "command" in tool_codes:
|
|
29
|
+
sections.append(
|
|
30
|
+
"Read references/<name> only when the Skill instructions require them. Skill scripts use paths "
|
|
31
|
+
"such as skills/<code>/scripts/<name> and may write generated text to scratch/."
|
|
32
|
+
)
|
|
33
|
+
sections.append("Tool results are facts: inspect errors and command exit_code before deciding the next action.")
|
|
34
|
+
if skills:
|
|
35
|
+
catalog = "\n".join(f"- `{skill.descriptor.code}`: {skill.descriptor.description}" for skill in skills)
|
|
36
|
+
sections.append(f"# Selected skills\n\n{catalog}")
|
|
37
|
+
return "\n\n".join(section for section in sections if section)
|
|
File without changes
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from enum import StrEnum
|
|
2
|
+
|
|
3
|
+
from pleiad.errors.base import BusinessError
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class CapabilityErrorCode(StrEnum):
|
|
7
|
+
"""Stable error codes owned by the capability component."""
|
|
8
|
+
|
|
9
|
+
NOT_FOUND = "capability.not_found"
|
|
10
|
+
DIGEST_MISMATCH = "capability.digest.mismatch"
|
|
11
|
+
UNAVAILABLE = "capability.unavailable"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class CapabilityError(BusinessError):
|
|
15
|
+
"""Base class for expected capability failures."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class CapabilityNotFoundError(CapabilityError):
|
|
19
|
+
"""Report a requested capability that does not exist."""
|
|
20
|
+
|
|
21
|
+
def __init__(self, message: str = "The requested capability does not exist.") -> None:
|
|
22
|
+
"""Create a missing capability error.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
message: Diagnostic description of the missing capability.
|
|
26
|
+
"""
|
|
27
|
+
super().__init__(CapabilityErrorCode.NOT_FOUND, message)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class CapabilityDigestMismatchError(CapabilityError):
|
|
31
|
+
"""Report a capability snapshot that changed before resume."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, message: str = "The capability snapshot changed before the Run resumed.") -> None:
|
|
34
|
+
"""Create a capability digest mismatch error.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
message: Diagnostic description of the mismatch.
|
|
38
|
+
"""
|
|
39
|
+
super().__init__(CapabilityErrorCode.DIGEST_MISMATCH, message)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class CapabilityUnavailableError(CapabilityError):
|
|
43
|
+
"""Report a known capability that cannot be used in the current Run."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, message: str = "The requested capability is unavailable for this Run.") -> None:
|
|
46
|
+
"""Create an unavailable capability error.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
message: Diagnostic description of the unavailable capability.
|
|
50
|
+
"""
|
|
51
|
+
super().__init__(CapabilityErrorCode.UNAVAILABLE, message)
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""Deterministic BM25 search for Tool and Skill descriptors."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from collections import Counter
|
|
5
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from math import log
|
|
8
|
+
from typing import Protocol
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class SearchableDescriptor(Protocol):
|
|
12
|
+
"""The public metadata shared by Tool and Skill descriptors."""
|
|
13
|
+
|
|
14
|
+
@property
|
|
15
|
+
def name(self) -> str:
|
|
16
|
+
"""Return the capability display name."""
|
|
17
|
+
...
|
|
18
|
+
|
|
19
|
+
@property
|
|
20
|
+
def code(self) -> str:
|
|
21
|
+
"""Return the stable capability identity."""
|
|
22
|
+
...
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def description(self) -> str:
|
|
26
|
+
"""Return searchable capability metadata."""
|
|
27
|
+
...
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
K1 = 1.2
|
|
31
|
+
B = 0.75
|
|
32
|
+
FIELD_WEIGHTS: Mapping[str, float] = {
|
|
33
|
+
"name": 4.0,
|
|
34
|
+
"description": 2.0,
|
|
35
|
+
"code": 1.0,
|
|
36
|
+
}
|
|
37
|
+
LATIN_OR_DIGIT = re.compile(r"[A-Za-z0-9_]+")
|
|
38
|
+
CJK = re.compile(r"[\u4e00-\u9fff]+")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True, slots=True)
|
|
42
|
+
class Document[DescriptorT: SearchableDescriptor]:
|
|
43
|
+
"""One immutable descriptor projection held by the search index."""
|
|
44
|
+
|
|
45
|
+
descriptor: DescriptorT
|
|
46
|
+
fields: Mapping[str, tuple[str, ...]]
|
|
47
|
+
term_frequencies: Mapping[str, Mapping[str, int]]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class BM25Index[DescriptorT: SearchableDescriptor]:
|
|
51
|
+
"""Rank descriptors with a small, in-memory, field-weighted BM25 index.
|
|
52
|
+
|
|
53
|
+
The index is intentionally rebuilt from a validated registry snapshot instead
|
|
54
|
+
of being persisted separately. Capability catalogs are already dynamic,
|
|
55
|
+
and rebuilding this metadata-only index on a catalog replacement keeps the
|
|
56
|
+
search view consistent with the descriptors and avoids a second storage
|
|
57
|
+
lifecycle for a derived artifact.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def __init__(self, descriptors: Iterable[DescriptorT]) -> None:
|
|
61
|
+
self.documents: tuple[Document[DescriptorT], ...] = tuple(
|
|
62
|
+
build_document(descriptor) for descriptor in descriptors
|
|
63
|
+
)
|
|
64
|
+
self.postings = self.build_postings()
|
|
65
|
+
self.document_frequency = {
|
|
66
|
+
field: {term: len(document_ids) for term, document_ids in terms.items()}
|
|
67
|
+
for field, terms in self.postings.items()
|
|
68
|
+
}
|
|
69
|
+
self.average_field_lengths = self.build_average_field_lengths()
|
|
70
|
+
|
|
71
|
+
def search(self, query: str = "", limit: int = 20) -> tuple[DescriptorT, ...]:
|
|
72
|
+
"""Return the highest-scoring descriptors in deterministic order."""
|
|
73
|
+
if limit < 1:
|
|
74
|
+
return ()
|
|
75
|
+
terms = tuple(dict.fromkeys(tokenize(query)))
|
|
76
|
+
if not terms:
|
|
77
|
+
descriptors = sorted(
|
|
78
|
+
(document.descriptor for document in self.documents),
|
|
79
|
+
key=descriptor_sort_key,
|
|
80
|
+
)
|
|
81
|
+
return tuple(descriptors[:limit])
|
|
82
|
+
|
|
83
|
+
candidate_ids = {
|
|
84
|
+
document_id
|
|
85
|
+
for field in FIELD_WEIGHTS
|
|
86
|
+
for term in terms
|
|
87
|
+
for document_id in self.postings[field].get(term, ())
|
|
88
|
+
}
|
|
89
|
+
ranked = [
|
|
90
|
+
(
|
|
91
|
+
self.score(self.documents[document_id], terms),
|
|
92
|
+
self.documents[document_id].descriptor,
|
|
93
|
+
)
|
|
94
|
+
for document_id in sorted(candidate_ids)
|
|
95
|
+
]
|
|
96
|
+
ranked = [item for item in ranked if item[0] > 0.0]
|
|
97
|
+
ranked.sort(key=lambda item: (-item[0], *descriptor_sort_key(item[1])))
|
|
98
|
+
return tuple(descriptor for _, descriptor in ranked[:limit])
|
|
99
|
+
|
|
100
|
+
def score(
|
|
101
|
+
self,
|
|
102
|
+
document: Document[DescriptorT],
|
|
103
|
+
terms: Sequence[str],
|
|
104
|
+
) -> float:
|
|
105
|
+
score = 0.0
|
|
106
|
+
for field, weight in FIELD_WEIGHTS.items():
|
|
107
|
+
tokens = document.fields[field]
|
|
108
|
+
if not tokens:
|
|
109
|
+
continue
|
|
110
|
+
average_length = self.average_field_lengths[field]
|
|
111
|
+
normalization = B * len(tokens) / average_length if average_length else 0.0
|
|
112
|
+
for term in terms:
|
|
113
|
+
frequency = document.term_frequencies[field].get(term, 0)
|
|
114
|
+
if not frequency:
|
|
115
|
+
continue
|
|
116
|
+
score += weight * self.term_score(
|
|
117
|
+
field,
|
|
118
|
+
term,
|
|
119
|
+
frequency,
|
|
120
|
+
normalization,
|
|
121
|
+
)
|
|
122
|
+
return score
|
|
123
|
+
|
|
124
|
+
def term_score(
|
|
125
|
+
self,
|
|
126
|
+
field: str,
|
|
127
|
+
term: str,
|
|
128
|
+
frequency: int,
|
|
129
|
+
normalization: float,
|
|
130
|
+
) -> float:
|
|
131
|
+
document_count = len(self.documents)
|
|
132
|
+
document_frequency = self.document_frequency[field][term]
|
|
133
|
+
inverse_frequency = log(1.0 + (document_count - document_frequency + 0.5) / (document_frequency + 0.5))
|
|
134
|
+
return inverse_frequency * (frequency * (K1 + 1.0) / (frequency + K1 * (1.0 - B + normalization)))
|
|
135
|
+
|
|
136
|
+
def build_postings(self) -> dict[str, dict[str, tuple[int, ...]]]:
|
|
137
|
+
"""Build field-specific postings lists for candidate retrieval."""
|
|
138
|
+
postings: dict[str, dict[str, list[int]]] = {field: {} for field in FIELD_WEIGHTS}
|
|
139
|
+
for document_id, document in enumerate(self.documents):
|
|
140
|
+
for field, frequencies in document.term_frequencies.items():
|
|
141
|
+
for term in frequencies:
|
|
142
|
+
postings[field].setdefault(term, []).append(document_id)
|
|
143
|
+
return {
|
|
144
|
+
field: {term: tuple(document_ids) for term, document_ids in terms.items()}
|
|
145
|
+
for field, terms in postings.items()
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
def build_average_field_lengths(self) -> dict[str, float]:
|
|
149
|
+
document_count = len(self.documents) or 1
|
|
150
|
+
return {
|
|
151
|
+
field: sum(len(document.fields[field]) for document in self.documents) / document_count
|
|
152
|
+
for field in FIELD_WEIGHTS
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def tokenize(value: str) -> tuple[str, ...]:
|
|
157
|
+
"""Tokenize mixed Chinese, Latin, numeric, and namespaced code text."""
|
|
158
|
+
tokens: list[str] = []
|
|
159
|
+
for match in LATIN_OR_DIGIT.finditer(value.casefold()):
|
|
160
|
+
token = match.group()
|
|
161
|
+
tokens.append(token)
|
|
162
|
+
if "_" in token:
|
|
163
|
+
tokens.extend(part for part in token.split("_") if part)
|
|
164
|
+
for match in CJK.finditer(value):
|
|
165
|
+
text = match.group()
|
|
166
|
+
if len(text) == 1:
|
|
167
|
+
tokens.append(text)
|
|
168
|
+
continue
|
|
169
|
+
tokens.extend(text[index : index + 2] for index in range(len(text) - 1))
|
|
170
|
+
return tuple(tokens)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def build_document[DescriptorT: SearchableDescriptor](descriptor: DescriptorT) -> Document[DescriptorT]:
|
|
174
|
+
fields = descriptor_fields(descriptor)
|
|
175
|
+
return Document(
|
|
176
|
+
descriptor=descriptor,
|
|
177
|
+
fields=fields,
|
|
178
|
+
term_frequencies={field: dict(Counter(tokens)) for field, tokens in fields.items()},
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def descriptor_fields(descriptor: SearchableDescriptor) -> dict[str, tuple[str, ...]]:
|
|
183
|
+
"""Project any Tool- or Skill-shaped descriptor into indexed fields."""
|
|
184
|
+
return {
|
|
185
|
+
"name": tokenize(descriptor.name),
|
|
186
|
+
"description": tokenize(descriptor.description),
|
|
187
|
+
"code": tokenize(descriptor.code),
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def descriptor_sort_key(descriptor: SearchableDescriptor) -> tuple[str, str]:
|
|
192
|
+
"""Return a stable presentation order for a searchable descriptor."""
|
|
193
|
+
return descriptor.name.casefold(), descriptor.code
|
|
File without changes
|
pleiad/context/budget.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Protocol
|
|
5
|
+
|
|
6
|
+
from pydantic import ConfigDict, Field, model_validator
|
|
7
|
+
|
|
8
|
+
from pleiad.context.errors import ContextWindowError
|
|
9
|
+
from pleiad.context.selection.models import ContextView
|
|
10
|
+
from pleiad.contracts import ContractModel
|
|
11
|
+
from pleiad.messages.message import Message
|
|
12
|
+
from pleiad.models.contracts import Tool
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True, slots=True)
|
|
18
|
+
class ContextCountInput:
|
|
19
|
+
"""Provider-neutral values needed to estimate one complete model request."""
|
|
20
|
+
|
|
21
|
+
instructions: str
|
|
22
|
+
messages: tuple[Message, ...]
|
|
23
|
+
tools: tuple[Tool, ...]
|
|
24
|
+
context_view: ContextView
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class TokenCounter(Protocol):
|
|
28
|
+
"""Count the input tokens for a provider-neutral model request."""
|
|
29
|
+
|
|
30
|
+
estimated: bool
|
|
31
|
+
|
|
32
|
+
def count(self, request: ContextCountInput) -> int:
|
|
33
|
+
"""Return a non-negative token count."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class ApproximateTokenCounter:
|
|
37
|
+
"""Provide a deterministic request-size estimate without a provider SDK."""
|
|
38
|
+
|
|
39
|
+
estimated = True
|
|
40
|
+
|
|
41
|
+
def count(self, request: ContextCountInput) -> int:
|
|
42
|
+
"""Estimate tokens from canonical JSON size."""
|
|
43
|
+
payload = {
|
|
44
|
+
"instructions": request.instructions,
|
|
45
|
+
"messages": [message.model_dump(mode="json") for message in request.messages],
|
|
46
|
+
"tools": [tool.model_dump(mode="json") for tool in request.tools],
|
|
47
|
+
}
|
|
48
|
+
serialized = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
49
|
+
return max(1, (len(serialized) + 3) // 4)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ContextBudget(ContractModel):
|
|
53
|
+
"""Explicit model-window and history-compaction limits."""
|
|
54
|
+
|
|
55
|
+
model_config = ConfigDict(strict=True)
|
|
56
|
+
|
|
57
|
+
window_tokens: int = Field(gt=0)
|
|
58
|
+
output_reserve: int = Field(gt=0)
|
|
59
|
+
compact_at: int = Field(gt=0)
|
|
60
|
+
recent_tokens: int = Field(ge=0)
|
|
61
|
+
|
|
62
|
+
@model_validator(mode="after")
|
|
63
|
+
def validate_limits(self) -> "ContextBudget":
|
|
64
|
+
"""Ensure all limits leave room for the reserved model output."""
|
|
65
|
+
if self.output_reserve >= self.window_tokens:
|
|
66
|
+
raise ValueError("output_reserve must be smaller than window_tokens")
|
|
67
|
+
if self.compact_at >= self.window_tokens - self.output_reserve:
|
|
68
|
+
raise ValueError("compact_at must be below the available input window")
|
|
69
|
+
if self.recent_tokens >= self.compact_at:
|
|
70
|
+
raise ValueError("recent_tokens must be smaller than compact_at")
|
|
71
|
+
return self
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def available_tokens(self) -> int:
|
|
75
|
+
"""Return the input capacity after reserving output tokens."""
|
|
76
|
+
return self.window_tokens - self.output_reserve
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(frozen=True, slots=True)
|
|
80
|
+
class BudgetStatus:
|
|
81
|
+
"""Result of evaluating one complete request against a ContextBudget."""
|
|
82
|
+
|
|
83
|
+
input_tokens: int
|
|
84
|
+
available_tokens: int
|
|
85
|
+
requires_compaction: bool
|
|
86
|
+
exceeds_window: bool
|
|
87
|
+
estimated: bool
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class BudgetGuard:
|
|
91
|
+
"""Evaluate request-local context budgets without mutating their inputs."""
|
|
92
|
+
|
|
93
|
+
def __init__(self, budget: ContextBudget, counter: TokenCounter | None = None) -> None:
|
|
94
|
+
"""Create a guard for one explicit budget and token counter."""
|
|
95
|
+
self.budget = budget
|
|
96
|
+
self.counter = ApproximateTokenCounter() if counter is None else counter
|
|
97
|
+
|
|
98
|
+
def check(self, request: ContextCountInput) -> BudgetStatus:
|
|
99
|
+
"""Return token and compaction facts for a complete request."""
|
|
100
|
+
input_tokens = self.counter.count(request)
|
|
101
|
+
if input_tokens < 0:
|
|
102
|
+
raise ValueError("token counters must return non-negative values")
|
|
103
|
+
status = BudgetStatus(
|
|
104
|
+
input_tokens=input_tokens,
|
|
105
|
+
available_tokens=self.budget.available_tokens,
|
|
106
|
+
requires_compaction=input_tokens >= self.budget.compact_at,
|
|
107
|
+
exceeds_window=input_tokens > self.budget.available_tokens,
|
|
108
|
+
estimated=self.counter.estimated,
|
|
109
|
+
)
|
|
110
|
+
logger.debug(
|
|
111
|
+
f"context budget input_tokens={status.input_tokens} available_tokens={status.available_tokens} "
|
|
112
|
+
f"estimated={status.estimated} requires_compaction={status.requires_compaction}"
|
|
113
|
+
)
|
|
114
|
+
return status
|
|
115
|
+
|
|
116
|
+
def ensure_fit(self, request: ContextCountInput) -> BudgetStatus:
|
|
117
|
+
"""Return the status or raise when required request content cannot fit."""
|
|
118
|
+
status = self.check(request)
|
|
119
|
+
if status.exceeds_window:
|
|
120
|
+
raise ContextWindowError("The required model input exceeds the available context window.")
|
|
121
|
+
return status
|
|
File without changes
|