tdqs 0.1.0__py3-none-any.whl

tdqs/__init__.py

@@ -0,0 +1,153 @@
+"""The reference implementation of the Tool Definition Quality Score, for Python.
+
+The package's public surface. The modules are one export each; this is the one stable entry
+point, so `from tdqs import score_server` keeps working when a module moves.
+"""
+
+from tdqs.build_server_coherence_prompt import CoherencePromptTool, build_server_coherence_prompt
+from tdqs.build_tool_scoring_prompt import build_tool_scoring_prompt
+from tdqs.compute_context_signals import compute_context_signals
+from tdqs.compute_input_hash import compute_input_hash
+from tdqs.compute_invocation_cost import compute_invocation_cost
+from tdqs.compute_smells import compute_smells
+from tdqs.compute_tdqs import compute_tdqs
+from tdqs.compute_tier import compute_tier
+from tdqs.context_signals import AnnotationValues, ContextSignals
+from tdqs.create_llm_client import OpenAiCompatibleLlmClient, create_llm_client
+from tdqs.evaluate_hard_gates import HardGateResult, evaluate_hard_gates
+from tdqs.find_shadow_candidates import CostedTool, ShadowCandidate, find_shadow_candidates
+from tdqs.format_lint_report_markdown import format_lint_report_markdown
+from tdqs.format_score_report_markdown import format_score_report_markdown
+from tdqs.lint_report import LintedTool, LintFinding, LintReport, ServerSummary
+from tdqs.lint_rules import LINT_RULES, LintRule, LintRuleKey, LintSeverity
+from tdqs.lint_server import lint_server
+from tdqs.llm_client import LlmClient
+from tdqs.load_tool_definitions import load_tool_definitions
+from tdqs.package_version import PACKAGE_VERSION
+from tdqs.parse_tool_definitions import ParsedToolDefinitions, parse_tool_definitions
+from tdqs.prompts.server_coherence_system_prompt import SERVER_COHERENCE_SYSTEM_PROMPT
+from tdqs.prompts.tool_scoring_system_prompt import TOOL_SCORING_SYSTEM_PROMPT
+from tdqs.request_hosted_report import HostedReport, HostedReportStatus, request_hosted_report
+from tdqs.rollup_server_score import rollup_server_score
+from tdqs.round1 import round1
+from tdqs.score_report import ScoreReport
+from tdqs.score_server import score_server
+from tdqs.score_server_coherence import ServerCoherence, score_server_coherence
+from tdqs.score_tool_definition import score_tool_definition
+from tdqs.serialize_tool_definition import serialize_tool_definition
+from tdqs.server_score import (
+    CoherenceJustifications,
+    CoherenceScores,
+    ServerScore,
+    ShadowingRisk,
+)
+from tdqs.spec.coherence_dimensions import (
+    COHERENCE_DIMENSIONS,
+    CoherenceDimension,
+    CoherenceDimensionKey,
+)
+from tdqs.spec.flags import FLAGS, Flag, HardGateFlag, ServerFlag, ToolFlag
+from tdqs.spec.spec_version import SPEC_VERSION
+from tdqs.spec.tiers import TIERS, Tier, TierDefinition
+from tdqs.spec.tool_dimensions import (
+    TOOL_DIMENSIONS,
+    DimensionScore,
+    ToolDimension,
+    ToolDimensionKey,
+)
+from tdqs.stringify_json import stringify_json
+from tdqs.tdqs_input_error import TdqsInputError
+from tdqs.tdqs_llm_error import TdqsLlmError
+from tdqs.tool_definition import ToolDefinition
+from tdqs.tool_definitions_source import (
+    CommandSource,
+    FileSource,
+    ToolDefinitionsSource,
+    UrlSource,
+)
+from tdqs.tool_score import DimensionJustifications, DimensionScores, Justification, ToolScore
+from tdqs.traverse_required_subtree import RequiredSubtreeSignals, traverse_required_subtree
+
+__all__ = [
+    "COHERENCE_DIMENSIONS",
+    "FLAGS",
+    "LINT_RULES",
+    "PACKAGE_VERSION",
+    "SERVER_COHERENCE_SYSTEM_PROMPT",
+    "SPEC_VERSION",
+    "TIERS",
+    "TOOL_DIMENSIONS",
+    "TOOL_SCORING_SYSTEM_PROMPT",
+    "AnnotationValues",
+    "CoherenceDimension",
+    "CoherenceDimensionKey",
+    "CoherenceJustifications",
+    "CoherencePromptTool",
+    "CoherenceScores",
+    "CommandSource",
+    "ContextSignals",
+    "CostedTool",
+    "DimensionJustifications",
+    "DimensionScore",
+    "DimensionScores",
+    "FileSource",
+    "Flag",
+    "HardGateFlag",
+    "HardGateResult",
+    "HostedReport",
+    "HostedReportStatus",
+    "Justification",
+    "LintFinding",
+    "LintReport",
+    "LintRule",
+    "LintRuleKey",
+    "LintSeverity",
+    "LintedTool",
+    "LlmClient",
+    "OpenAiCompatibleLlmClient",
+    "ParsedToolDefinitions",
+    "RequiredSubtreeSignals",
+    "ScoreReport",
+    "ServerCoherence",
+    "ServerFlag",
+    "ServerScore",
+    "ServerSummary",
+    "ShadowCandidate",
+    "ShadowingRisk",
+    "TdqsInputError",
+    "TdqsLlmError",
+    "Tier",
+    "TierDefinition",
+    "ToolDefinition",
+    "ToolDefinitionsSource",
+    "ToolDimension",
+    "ToolDimensionKey",
+    "ToolFlag",
+    "ToolScore",
+    "UrlSource",
+    "build_server_coherence_prompt",
+    "build_tool_scoring_prompt",
+    "compute_context_signals",
+    "compute_input_hash",
+    "compute_invocation_cost",
+    "compute_smells",
+    "compute_tdqs",
+    "compute_tier",
+    "create_llm_client",
+    "evaluate_hard_gates",
+    "find_shadow_candidates",
+    "format_lint_report_markdown",
+    "format_score_report_markdown",
+    "lint_server",
+    "load_tool_definitions",
+    "parse_tool_definitions",
+    "request_hosted_report",
+    "rollup_server_score",
+    "round1",
+    "score_server",
+    "score_server_coherence",
+    "score_tool_definition",
+    "serialize_tool_definition",
+    "stringify_json",
+    "traverse_required_subtree",
+]

tdqs/__main__.py

@@ -0,0 +1,3 @@
+from tdqs.cli.main import main
+
+main()

tdqs/build_server_coherence_prompt.py

@@ -0,0 +1,63 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from tdqs.context_signals import ContextSignals
+from tdqs.find_shadow_candidates import ShadowCandidate
+from tdqs.sanitize_for_prompt import sanitize_for_prompt
+
+
+@dataclass(frozen=True)
+class CoherencePromptTool:
+    context_signals: ContextSignals
+    description: str | None
+    name: str
+
+
+def build_server_coherence_prompt(
+    *,
+    server_name: str,
+    shadow_candidates: Sequence[ShadowCandidate],
+    tools: Sequence[CoherencePromptTool],
+) -> str:
+    """The user message of the coherence call, in the specification's template: every tool with
+    its invocation cost and the three signals behind it, then the candidate pairs the prefilter
+    emitted. The model is told to judge only those pairs, which is why they are spelled out rather
+    than left for it to find."""
+    tool_lines: list[str] = []
+
+    for tool in tools:
+        signals = tool.context_signals
+        cost = (
+            f"cost {signals.invocation_cost}: {signals.required_field_count} required, "
+            f"depth {signals.schema_depth}, {signals.union_choice_count} union choices"
+        )
+        text = (
+            "(no description)"
+            if tool.description is None
+            else str(sanitize_for_prompt(tool.description))
+        )
+
+        tool_lines.append(f"- {tool.name} [{cost}]: {text}")
+
+    candidate_lines = [
+        f"{candidate.tool} (cost {candidate.invocation_cost}) may be shadowed by "
+        f"{candidate.cheaper_sibling} (cost {candidate.cheaper_sibling_invocation_cost})"
+        for candidate in shadow_candidates
+    ]
+
+    return "\n".join(
+        [
+            f"SERVER NAME: {sanitize_for_prompt(server_name)}",
+            f"TOOL COUNT: {len(tools)}",
+            "",
+            "<tools>",
+            *tool_lines,
+            "</tools>",
+            "",
+            "<shadow-candidates>",
+            "\n".join(candidate_lines) if len(candidate_lines) > 0 else "None",
+            "</shadow-candidates>",
+            "",
+            "Respond with JSON only.",
+        ]
+    )

tdqs/build_tool_scoring_prompt.py

@@ -0,0 +1,61 @@
+from collections.abc import Sequence
+
+from tdqs.context_signals import ContextSignals
+from tdqs.sanitize_for_prompt import sanitize_for_prompt
+from tdqs.stringify_json import stringify_json
+from tdqs.tool_definition import ToolDefinition
+
+
+def _js_boolean(value: bool) -> str:
+    return "true" if value else "false"
+
+
+def build_tool_scoring_prompt(
+    *, context_signals: ContextSignals, sibling_tool_names: Sequence[str], tool: ToolDefinition
+) -> str:
+    """The user message of the tool-scoring call, in the specification's template. Schemas are
+    pretty-printed in their own key order — the deterministic serialization is for hashing, and
+    the model reads the schema the way its author wrote it. The invocation-cost signals are
+    deliberately absent: the specification withholds them from this prompt."""
+    input_schema = stringify_json(
+        sanitize_for_prompt(tool.input_schema if tool.input_schema is not None else {}), indent=2
+    )
+    annotations = (
+        "None provided"
+        if tool.annotations is None
+        else stringify_json(sanitize_for_prompt(tool.annotations), indent=2)
+    )
+    siblings = "\n".join(sibling_tool_names) if len(sibling_tool_names) > 0 else "None"
+    description = sanitize_for_prompt(tool.description if tool.description is not None else "")
+
+    return "\n".join(
+        [
+            f"TOOL NAME: {tool.name}",
+            f"TITLE: {tool.title if tool.title is not None else 'null'}",
+            "",
+            "DESCRIPTION:",
+            f'"{description}"',
+            "",
+            "<input-schema>",
+            input_schema,
+            "</input-schema>",
+            "",
+            "<annotations>",
+            annotations,
+            "</annotations>",
+            "",
+            "CONTEXT SIGNALS:",
+            f"- Parameter count: {context_signals.param_count}",
+            f"- Required parameters: {context_signals.required_param_count}",
+            f"- Schema description coverage: {context_signals.schema_description_coverage}%",
+            f"- Parameters with enums: {context_signals.params_with_enums}",
+            f"- Has output schema: {_js_boolean(context_signals.has_output_schema)}",
+            f"- Has nested objects: {_js_boolean(context_signals.has_nested_objects)}",
+            "",
+            "<sibling-tools>",
+            siblings,
+            "</sibling-tools>",
+            "",
+            "Respond with JSON only.",
+        ]
+    )

tdqs/camel_model.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, ConfigDict
+from pydantic.alias_generators import to_camel
+
+
+class CamelModel(BaseModel):
+    """A report model whose JSON is the specification's: camelCase keys, in the field order the
+    class declares. Attributes stay snake_case, and both spellings are accepted on the way in, so
+    a report read back from `tdqs score --format json` and one built in Python validate alike.
+    Dimension-keyed maps are `TypedDict`s rather than models because their keys are snake_case in
+    the specification and must not be renamed."""
+
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        serialize_by_alias=True,
+        validate_by_alias=True,
+        validate_by_name=True,
+    )

tdqs/cli/__init__.py

@@ -0,0 +1 @@
+"""The `tdqs` command: argument parsing, the two subcommands and their terminal renderings."""

tdqs/cli/cli_usage.py

@@ -0,0 +1,31 @@
+CLI_USAGE = """tdqs — Tool Definition Quality Score
+
+Usage
+  tdqs lint  [input] [--server-name <name>] [--format json|markdown|text] [--fail-on error|warning|never] [--output <path>]
+  tdqs score [input] [--model <id>] [--base-url <url>] [--api-key <key>] [--concurrency <n>]
+             [--request-overrides <json>] [--format json|markdown|text] [--fail-under A|B|C|D]
+             [--output <path>] [--hosted <site url>]
+  tdqs --version
+  tdqs --help
+
+Input (exactly one)
+  --file <path>        A JSON file: a tools/list result ({ "tools": [...] }), an array of tools, or one tool.
+                       "-" reads standard input.
+  --command "<cmd>"    Spawn a stdio MCP server and read its tools/list.
+  --url <url>          Connect to a Streamable HTTP MCP server. Repeat --header "Name: value" for auth.
+
+lint    The deterministic stages only: context signals, hard gates, the shadow prefilter and the
+        checklist the specification ranks highest. Needs no model and no key.
+score   The full pipeline: every tool through the rubric, then coherence, then the server rollup.
+        Needs an OpenAI-compatible endpoint (--base-url, --api-key, --model, or the TDQS_BASE_URL,
+        TDQS_API_KEY and TDQS_MODEL environment variables), or --hosted to score through a TDQS site
+        with that site's API key as --api-key or TDQS_API_KEY.
+        --request-overrides (or TDQS_REQUEST_OVERRIDES) merges provider-specific JSON into every
+        request; turn extended reasoning off with it, e.g. '{"reasoning":{"enabled":false}}' on
+        OpenRouter — the calibration examples reproduce with reasoning off, at seconds per call.
+
+Exit codes
+  0  done
+  1  the threshold was not met (--fail-on for lint, --fail-under for score)
+  2  usage error, unreadable input, unreachable server, or a model failure
+"""

tdqs/cli/format_lint_report_text.py

@@ -0,0 +1,41 @@
+from tdqs.lint_report import LintReport
+from tdqs.lint_rules import LintSeverity
+
+
+def _yes_no(value: bool) -> str:
+    return "yes" if value else "no"
+
+
+def format_lint_report_text(report: LintReport) -> str:
+    """The terminal rendering: one line per finding, grouped by tool, terse enough to read in a CI
+    log. The markdown and JSON formats carry everything; this one carries what a person scans."""
+    lines: list[str] = [
+        (
+            f"TDQS lint · {report.server.name} · {report.server.tool_count} tools · specification "
+            f"{report.spec_version}"
+        ),
+        "",
+    ]
+
+    for tool in report.tools:
+        signals = tool.context_signals
+
+        lines.append(
+            f"{tool.name}  params {signals.param_count} · coverage "
+            f"{signals.schema_description_coverage}% · annotations "
+            f"{_yes_no(signals.has_annotations)} · output schema "
+            f"{_yes_no(signals.has_output_schema)} · cost {signals.invocation_cost}"
+        )
+        lines.extend(
+            f"  {finding.severity.ljust(7)} {finding.rule}: {finding.message}"
+            for finding in tool.findings
+        )
+
+    def count(severity: LintSeverity) -> int:
+        return sum(1 for finding in report.findings if finding.severity == severity)
+
+    lines.extend(
+        ["", f"{count('error')} errors, {count('warning')} warnings, {count('info')} notes"]
+    )
+
+    return "\n".join(lines) + "\n"

tdqs/cli/format_score_report_text.py

@@ -0,0 +1,41 @@
+from tdqs.score_report import ScoreReport
+from tdqs.spec.tool_dimensions import TOOL_DIMENSIONS
+
+
+def format_score_report_text(report: ScoreReport) -> str:
+    """The terminal rendering of a score: the rollup, then one line per tool with its six
+    dimension scores in rubric order. Justifications live in the markdown and JSON formats."""
+    server_score = report.server_score
+    model = "" if report.model is None else f" · model {report.model}"
+    headings = " ".join(dimension.key.split("_")[0][:8].ljust(8) for dimension in TOOL_DIMENSIONS)
+    lines: list[str] = [
+        (
+            f"TDQS {server_score.overall_tier} {server_score.overall_score:.1f} · "
+            f"{report.server.name} · {report.server.tool_count} tools"
+        ),
+        (
+            f"  tool definition quality {server_score.description_quality_tier} "
+            f"{server_score.description_quality_score:.1f} (mean {server_score.mean_tdqs:.1f}, "
+            f"min {server_score.min_tdqs:.1f}) · coherence {server_score.coherence_tier} "
+            f"{server_score.coherence_score:.1f}"
+        ),
+        f"  specification {report.spec_version}{model}",
+        "",
+        f"  {'tool'.ljust(40)} tdqs  {headings} flags",
+    ]
+
+    for tool in report.tools:
+        scores = " ".join(str(tool.scores[dimension.key]).ljust(8) for dimension in TOOL_DIMENSIONS)
+        flags = ", ".join([*tool.flags, *tool.server_flags])
+
+        lines.append(f"{tool.tier} {tool.name[:40].ljust(40)} {tool.tdqs:.1f}   {scores} {flags}")
+
+    for risk in server_score.shadowing_risks:
+        lines.append("")
+        lines.append(
+            f"shadowing risk: {risk.tool} (cost {risk.invocation_cost}) may be shadowed by "
+            f"{risk.cheaper_sibling} (cost {risk.cheaper_sibling_invocation_cost}) — "
+            f"{risk.justification}"
+        )
+
+    return "\n".join(lines) + "\n"

tdqs/cli/main.py

@@ -0,0 +1,10 @@
+import os
+import sys
+
+from tdqs.cli.run_cli import run_cli
+
+
+def main() -> None:
+    """The one place the process boundary is crossed: argv, environment, the two streams and the
+    exit code. Everything behind `run_cli` is testable in-process."""
+    sys.exit(run_cli(argv=sys.argv[1:], env=os.environ, stderr=sys.stderr, stdout=sys.stdout))